2024-05-09T03:34:37.217 INFO:root:teuthology version: 0.0.1.dev271+g1ae7ad8 2024-05-09T03:34:37.217 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492 --name adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi --description orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} --owner scheduled_adking@teuthology --verbose -- /home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492/orig.config.yaml 2024-05-09T03:34:37.263 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-09T03:34:37.353 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492 branch: wip-adk-testing-2024-05-08-1927 description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: adking@redhat.com first_in_suite: false job_id: '7699492' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk-testing-2024-05-08-1927 ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_STRAY_DAEMON log-only-match: - CEPHADM_ sha1: c66c553a84eb9faa0029f5f0bb8284e3a826ac85 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: c66c553a84eb9faa0029f5f0bb8284e3a826ac85 workunit: branch: wip-adk-testing-2024-05-08-1927 sha1: c66c553a84eb9faa0029f5f0bb8284e3a826ac85 owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 4873 sha1: c66c553a84eb9faa0029f5f0bb8284e3a826ac85 sleep_before_teardown: 0 subset: 1/20 suite: orch:cephadm suite_branch: wip-adk-testing-2024-05-08-1927 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_c66c553a84eb9faa0029f5f0bb8284e3a826ac85/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: c66c553a84eb9faa0029f5f0bb8284e3a826ac85 targets: smithi078.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHcBM+SsFuPZS1hO0Jlswm+6Gw/1lbNPUcdD03IbpUhCcYq+Vo0k7qQ1qrFnJdZGFfYSEMkrONbPu9h78DjztIw= smithi132.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKootgy+oHLu3wh4K4h2ucOE2hhkSS/TIF9xEI1QeTZFzii8lw02Hf0mTkQT/JEobk6WkJWeLNWWEdVIzburuRc= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 1ae7ad82388e92a475afff437d49054826c019a1 timestamp: 2024-05-09_03:09:40 tube: smithi user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-05-09T03:34:37.353 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_c66c553a84eb9faa0029f5f0bb8284e3a826ac85/qa; will attempt to use it 2024-05-09T03:34:37.354 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_c66c553a84eb9faa0029f5f0bb8284e3a826ac85/qa/tasks 2024-05-09T03:34:37.354 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-05-09T03:34:37.355 INFO:teuthology.task.internal:Checking packages... 2024-05-09T03:34:37.381 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'c66c553a84eb9faa0029f5f0bb8284e3a826ac85' 2024-05-09T03:34:37.381 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-05-09T03:34:37.381 INFO:teuthology.packaging:ref: None 2024-05-09T03:34:37.381 INFO:teuthology.packaging:tag: None 2024-05-09T03:34:37.381 INFO:teuthology.packaging:branch: wip-adk-testing-2024-05-08-1927 2024-05-09T03:34:37.381 INFO:teuthology.packaging:sha1: c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T03:34:37.381 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk-testing-2024-05-08-1927 2024-05-09T03:34:37.600 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3631.gc66c553a 2024-05-09T03:34:37.602 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-05-09T03:34:37.608 INFO:teuthology.task.internal:no buildpackages task found 2024-05-09T03:34:37.608 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-05-09T03:34:37.620 INFO:teuthology.task.internal:Saving configuration 2024-05-09T03:34:37.638 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-05-09T03:34:37.643 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-09T03:34:37.668 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi078.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-09 03:27:28.702707', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHcBM+SsFuPZS1hO0Jlswm+6Gw/1lbNPUcdD03IbpUhCcYq+Vo0k7qQ1qrFnJdZGFfYSEMkrONbPu9h78DjztIw='} 2024-05-09T03:34:37.694 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi132.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-09 03:27:28.701300', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKootgy+oHLu3wh4K4h2ucOE2hhkSS/TIF9xEI1QeTZFzii8lw02Hf0mTkQT/JEobk6WkJWeLNWWEdVIzburuRc='} 2024-05-09T03:34:37.694 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-05-09T03:34:37.700 INFO:teuthology.task.internal:roles: ubuntu@smithi078.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-05-09T03:34:37.700 INFO:teuthology.task.internal:roles: ubuntu@smithi132.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-05-09T03:34:37.700 INFO:teuthology.run_tasks:Running task console_log... 2024-05-09T03:34:37.786 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fa955c30160>, signals=[15]) 2024-05-09T03:34:37.786 INFO:teuthology.run_tasks:Running task internal.connect... 2024-05-09T03:34:37.792 INFO:teuthology.task.internal:Opening connections... 2024-05-09T03:34:37.793 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi078.front.sepia.ceph.com 2024-05-09T03:34:37.794 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-09T03:34:37.868 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi132.front.sepia.ceph.com 2024-05-09T03:34:37.869 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi132.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-09T03:34:37.945 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-05-09T03:34:37.952 DEBUG:teuthology.orchestra.run.smithi078:> uname -m 2024-05-09T03:34:37.968 INFO:teuthology.orchestra.run.smithi078.stdout:x86_64 2024-05-09T03:34:37.969 DEBUG:teuthology.orchestra.run.smithi078:> cat /etc/os-release 2024-05-09T03:34:38.025 INFO:teuthology.orchestra.run.smithi078.stdout:NAME="CentOS Stream" 2024-05-09T03:34:38.026 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION="9" 2024-05-09T03:34:38.026 INFO:teuthology.orchestra.run.smithi078.stdout:ID="centos" 2024-05-09T03:34:38.026 INFO:teuthology.orchestra.run.smithi078.stdout:ID_LIKE="rhel fedora" 2024-05-09T03:34:38.026 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION_ID="9" 2024-05-09T03:34:38.026 INFO:teuthology.orchestra.run.smithi078.stdout:PLATFORM_ID="platform:el9" 2024-05-09T03:34:38.026 INFO:teuthology.orchestra.run.smithi078.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-09T03:34:38.026 INFO:teuthology.orchestra.run.smithi078.stdout:ANSI_COLOR="0;31" 2024-05-09T03:34:38.026 INFO:teuthology.orchestra.run.smithi078.stdout:LOGO="fedora-logo-icon" 2024-05-09T03:34:38.027 INFO:teuthology.orchestra.run.smithi078.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-09T03:34:38.027 INFO:teuthology.orchestra.run.smithi078.stdout:HOME_URL="https://centos.org/" 2024-05-09T03:34:38.027 INFO:teuthology.orchestra.run.smithi078.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-09T03:34:38.027 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-09T03:34:38.027 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-09T03:34:38.027 INFO:teuthology.lock.ops:Updating smithi078.front.sepia.ceph.com on lock server 2024-05-09T03:34:38.049 DEBUG:teuthology.orchestra.run.smithi132:> uname -m 2024-05-09T03:34:38.067 INFO:teuthology.orchestra.run.smithi132.stdout:x86_64 2024-05-09T03:34:38.067 DEBUG:teuthology.orchestra.run.smithi132:> cat /etc/os-release 2024-05-09T03:34:38.124 INFO:teuthology.orchestra.run.smithi132.stdout:NAME="CentOS Stream" 2024-05-09T03:34:38.124 INFO:teuthology.orchestra.run.smithi132.stdout:VERSION="9" 2024-05-09T03:34:38.125 INFO:teuthology.orchestra.run.smithi132.stdout:ID="centos" 2024-05-09T03:34:38.125 INFO:teuthology.orchestra.run.smithi132.stdout:ID_LIKE="rhel fedora" 2024-05-09T03:34:38.125 INFO:teuthology.orchestra.run.smithi132.stdout:VERSION_ID="9" 2024-05-09T03:34:38.125 INFO:teuthology.orchestra.run.smithi132.stdout:PLATFORM_ID="platform:el9" 2024-05-09T03:34:38.125 INFO:teuthology.orchestra.run.smithi132.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-09T03:34:38.125 INFO:teuthology.orchestra.run.smithi132.stdout:ANSI_COLOR="0;31" 2024-05-09T03:34:38.125 INFO:teuthology.orchestra.run.smithi132.stdout:LOGO="fedora-logo-icon" 2024-05-09T03:34:38.125 INFO:teuthology.orchestra.run.smithi132.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-09T03:34:38.125 INFO:teuthology.orchestra.run.smithi132.stdout:HOME_URL="https://centos.org/" 2024-05-09T03:34:38.125 INFO:teuthology.orchestra.run.smithi132.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-09T03:34:38.125 INFO:teuthology.orchestra.run.smithi132.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-09T03:34:38.125 INFO:teuthology.orchestra.run.smithi132.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-09T03:34:38.126 INFO:teuthology.lock.ops:Updating smithi132.front.sepia.ceph.com on lock server 2024-05-09T03:34:38.148 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-05-09T03:34:38.157 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-05-09T03:34:38.164 INFO:teuthology.task.internal:Checking for old test directory... 2024-05-09T03:34:38.164 DEBUG:teuthology.orchestra.run.smithi078:> test '!' -e /home/ubuntu/cephtest 2024-05-09T03:34:38.168 DEBUG:teuthology.orchestra.run.smithi132:> test '!' -e /home/ubuntu/cephtest 2024-05-09T03:34:38.184 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-05-09T03:34:38.191 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-05-09T03:34:38.191 DEBUG:teuthology.orchestra.run.smithi078:> test -z $(ls -A /var/lib/ceph) 2024-05-09T03:34:38.224 DEBUG:teuthology.orchestra.run.smithi132:> test -z $(ls -A /var/lib/ceph) 2024-05-09T03:34:38.327 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-05-09T03:34:38.382 INFO:teuthology.run_tasks:Running task kernel... 2024-05-09T03:34:38.396 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-05-09T03:34:38.396 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-05-09T03:34:38.397 DEBUG:teuthology.orchestra.run.smithi078:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-09T03:34:38.397 DEBUG:teuthology.orchestra.run.smithi132:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-09T03:34:38.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-09T03:34:38.413 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-05-09T03:34:38.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-09T03:34:38.414 DEBUG:teuthology.orchestra.run.smithi132:> uname -r 2024-05-09T03:34:38.470 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-437.el9.x86_64 2024-05-09T03:34:38.470 INFO:teuthology.task.kernel:Running kernel on smithi078: 5.14.0-437.el9.x86_64 2024-05-09T03:34:38.471 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum install -y kernel 2024-05-09T03:34:38.472 INFO:teuthology.orchestra.run.smithi132.stdout:5.14.0-437.el9.x86_64 2024-05-09T03:34:38.473 INFO:teuthology.task.kernel:Running kernel on smithi132: 5.14.0-437.el9.x86_64 2024-05-09T03:34:38.473 DEBUG:teuthology.orchestra.run.smithi132:> sudo yum install -y kernel 2024-05-09T03:34:41.162 INFO:teuthology.orchestra.run.smithi132.stdout:CentOS Stream 9 - BaseOS 8.1 MB/s | 8.1 MB 00:00 2024-05-09T03:34:41.767 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - BaseOS 4.8 MB/s | 8.1 MB 00:01 2024-05-09T03:34:42.979 INFO:teuthology.orchestra.run.smithi132.stdout:CentOS Stream 9 - AppStream 26 MB/s | 19 MB 00:00 2024-05-09T03:34:43.693 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - AppStream 23 MB/s | 19 MB 00:00 2024-05-09T03:34:47.172 INFO:teuthology.orchestra.run.smithi132.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.2 MB 00:00 2024-05-09T03:34:48.619 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - CRB 4.7 MB/s | 6.2 MB 00:01 2024-05-09T03:34:48.730 INFO:teuthology.orchestra.run.smithi132.stdout:CentOS Stream 9 - Extras packages 74 kB/s | 16 kB 00:00 2024-05-09T03:34:49.791 INFO:teuthology.orchestra.run.smithi132.stdout:Extra Packages for Enterprise Linux 23 MB/s | 21 MB 00:00 2024-05-09T03:34:50.261 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - Extras packages 52 kB/s | 16 kB 00:00 2024-05-09T03:34:54.958 INFO:teuthology.orchestra.run.smithi132.stdout:lab-extras 26 kB/s | 1.7 kB 00:00 2024-05-09T03:34:56.997 INFO:teuthology.orchestra.run.smithi132.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-09T03:34:56.997 INFO:teuthology.orchestra.run.smithi132.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-09T03:34:57.088 INFO:teuthology.orchestra.run.smithi132.stdout:Dependencies resolved. 2024-05-09T03:34:57.097 INFO:teuthology.orchestra.run.smithi132.stdout:================================================================================ 2024-05-09T03:34:57.097 INFO:teuthology.orchestra.run.smithi132.stdout: Package Architecture Version Repository Size 2024-05-09T03:34:57.097 INFO:teuthology.orchestra.run.smithi132.stdout:================================================================================ 2024-05-09T03:34:57.097 INFO:teuthology.orchestra.run.smithi132.stdout:Installing: 2024-05-09T03:34:57.097 INFO:teuthology.orchestra.run.smithi132.stdout: kernel x86_64 5.14.0-444.el9 baseos 484 k 2024-05-09T03:34:57.097 INFO:teuthology.orchestra.run.smithi132.stdout:Installing dependencies: 2024-05-09T03:34:57.097 INFO:teuthology.orchestra.run.smithi132.stdout: kernel-core x86_64 5.14.0-444.el9 baseos 15 M 2024-05-09T03:34:57.097 INFO:teuthology.orchestra.run.smithi132.stdout: kernel-modules x86_64 5.14.0-444.el9 baseos 34 M 2024-05-09T03:34:57.097 INFO:teuthology.orchestra.run.smithi132.stdout: kernel-modules-core x86_64 5.14.0-444.el9 baseos 28 M 2024-05-09T03:34:57.097 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:34:57.097 INFO:teuthology.orchestra.run.smithi132.stdout:Transaction Summary 2024-05-09T03:34:57.097 INFO:teuthology.orchestra.run.smithi132.stdout:================================================================================ 2024-05-09T03:34:57.098 INFO:teuthology.orchestra.run.smithi132.stdout:Install 4 Packages 2024-05-09T03:34:57.098 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:34:57.098 INFO:teuthology.orchestra.run.smithi132.stdout:Total download size: 78 M 2024-05-09T03:34:57.098 INFO:teuthology.orchestra.run.smithi132.stdout:Installed size: 123 M 2024-05-09T03:34:57.098 INFO:teuthology.orchestra.run.smithi132.stdout:Downloading Packages: 2024-05-09T03:34:58.075 INFO:teuthology.orchestra.run.smithi132.stdout:(1/4): kernel-5.14.0-444.el9.x86_64.rpm 518 kB/s | 484 kB 00:00 2024-05-09T03:34:58.701 INFO:teuthology.orchestra.run.smithi132.stdout:(2/4): kernel-core-5.14.0-444.el9.x86_64.rpm 9.9 MB/s | 15 MB 00:01 2024-05-09T03:34:59.576 INFO:teuthology.orchestra.run.smithi132.stdout:(3/4): kernel-modules-5.14.0-444.el9.x86_64.rpm 14 MB/s | 34 MB 00:02 2024-05-09T03:35:00.401 INFO:teuthology.orchestra.run.smithi132.stdout:(4/4): kernel-modules-core-5.14.0-444.el9.x86_6 12 MB/s | 28 MB 00:02 2024-05-09T03:35:00.402 INFO:teuthology.orchestra.run.smithi132.stdout:-------------------------------------------------------------------------------- 2024-05-09T03:35:00.402 INFO:teuthology.orchestra.run.smithi132.stdout:Total 24 MB/s | 78 MB 00:03 2024-05-09T03:35:01.080 INFO:teuthology.orchestra.run.smithi132.stdout:Running transaction check 2024-05-09T03:35:01.399 INFO:teuthology.orchestra.run.smithi132.stdout:Transaction check succeeded. 2024-05-09T03:35:01.399 INFO:teuthology.orchestra.run.smithi132.stdout:Running transaction test 2024-05-09T03:35:02.320 INFO:teuthology.orchestra.run.smithi132.stdout:Transaction test succeeded. 2024-05-09T03:35:02.320 INFO:teuthology.orchestra.run.smithi132.stdout:Running transaction 2024-05-09T03:35:04.723 INFO:teuthology.orchestra.run.smithi132.stdout: Preparing : 1/1 2024-05-09T03:35:05.436 INFO:teuthology.orchestra.run.smithi132.stdout: Installing : kernel-modules-core-5.14.0-444.el9.x86_64 1/4 2024-05-09T03:35:05.519 INFO:teuthology.orchestra.run.smithi132.stdout: Installing : kernel-core-5.14.0-444.el9.x86_64 2/4 2024-05-09T03:35:08.200 INFO:teuthology.orchestra.run.smithi132.stdout: Running scriptlet: kernel-core-5.14.0-444.el9.x86_64 2/4 2024-05-09T03:35:08.326 INFO:teuthology.orchestra.run.smithi132.stdout: Installing : kernel-modules-5.14.0-444.el9.x86_64 3/4 2024-05-09T03:35:09.805 INFO:teuthology.orchestra.run.smithi078.stdout:Extra Packages for Enterprise Linux 1.1 MB/s | 21 MB 00:19 2024-05-09T03:35:13.037 INFO:teuthology.orchestra.run.smithi132.stdout: Running scriptlet: kernel-modules-5.14.0-444.el9.x86_64 3/4 2024-05-09T03:35:13.087 INFO:teuthology.orchestra.run.smithi132.stdout: Installing : kernel-5.14.0-444.el9.x86_64 4/4 2024-05-09T03:35:14.886 INFO:teuthology.orchestra.run.smithi078.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-05-09T03:35:16.884 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-09T03:35:16.884 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-09T03:35:16.974 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-05-09T03:35:16.985 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-05-09T03:35:16.985 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-05-09T03:35:16.985 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-05-09T03:35:16.985 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-05-09T03:35:16.985 INFO:teuthology.orchestra.run.smithi078.stdout: kernel x86_64 5.14.0-444.el9 baseos 484 k 2024-05-09T03:35:16.985 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-05-09T03:35:16.986 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core x86_64 5.14.0-444.el9 baseos 15 M 2024-05-09T03:35:16.986 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules x86_64 5.14.0-444.el9 baseos 34 M 2024-05-09T03:35:16.986 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core x86_64 5.14.0-444.el9 baseos 28 M 2024-05-09T03:35:16.986 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:35:16.986 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-05-09T03:35:16.986 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-05-09T03:35:16.986 INFO:teuthology.orchestra.run.smithi078.stdout:Install 4 Packages 2024-05-09T03:35:16.986 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:35:16.987 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 78 M 2024-05-09T03:35:16.987 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 123 M 2024-05-09T03:35:16.987 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-05-09T03:35:17.737 INFO:teuthology.orchestra.run.smithi078.stdout:(1/4): kernel-5.14.0-444.el9.x86_64.rpm 720 kB/s | 484 kB 00:00 2024-05-09T03:35:17.804 INFO:teuthology.orchestra.run.smithi132.stdout: Running scriptlet: kernel-modules-core-5.14.0-444.el9.x86_64 4/4 2024-05-09T03:35:32.278 INFO:teuthology.orchestra.run.smithi078.stdout:(2/4): kernel-core-5.14.0-444.el9.x86_64.rpm 1.0 MB/s | 15 MB 00:15 2024-05-09T03:35:46.971 INFO:teuthology.orchestra.run.smithi132.stdout: Running scriptlet: kernel-core-5.14.0-444.el9.x86_64 4/4 2024-05-09T03:35:47.009 INFO:teuthology.orchestra.run.smithi132.stdout: Running scriptlet: kernel-modules-5.14.0-444.el9.x86_64 4/4 2024-05-09T03:35:47.371 INFO:teuthology.orchestra.run.smithi132.stdout: Running scriptlet: kernel-5.14.0-444.el9.x86_64 4/4 2024-05-09T03:35:47.371 INFO:teuthology.orchestra.run.smithi132.stdout: Verifying : kernel-5.14.0-444.el9.x86_64 1/4 2024-05-09T03:35:47.371 INFO:teuthology.orchestra.run.smithi132.stdout: Verifying : kernel-core-5.14.0-444.el9.x86_64 2/4 2024-05-09T03:35:47.371 INFO:teuthology.orchestra.run.smithi132.stdout: Verifying : kernel-modules-5.14.0-444.el9.x86_64 3/4 2024-05-09T03:35:48.196 INFO:teuthology.orchestra.run.smithi132.stdout: Verifying : kernel-modules-core-5.14.0-444.el9.x86_64 4/4 2024-05-09T03:35:48.196 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:35:48.196 INFO:teuthology.orchestra.run.smithi132.stdout:Installed: 2024-05-09T03:35:48.196 INFO:teuthology.orchestra.run.smithi132.stdout: kernel-5.14.0-444.el9.x86_64 2024-05-09T03:35:48.196 INFO:teuthology.orchestra.run.smithi132.stdout: kernel-core-5.14.0-444.el9.x86_64 2024-05-09T03:35:48.196 INFO:teuthology.orchestra.run.smithi132.stdout: kernel-modules-5.14.0-444.el9.x86_64 2024-05-09T03:35:48.196 INFO:teuthology.orchestra.run.smithi132.stdout: kernel-modules-core-5.14.0-444.el9.x86_64 2024-05-09T03:35:48.196 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:35:48.196 INFO:teuthology.orchestra.run.smithi132.stdout:Complete! 2024-05-09T03:35:48.581 DEBUG:teuthology.orchestra.run.smithi132:> rpm -q kernel | sort -rV | head -n 1 2024-05-09T03:35:48.649 INFO:teuthology.orchestra.run.smithi132.stdout:kernel-5.14.0-444.el9.x86_64 2024-05-09T03:35:48.649 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-444.el9.x86_64 2024-05-09T03:35:48.649 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-444.el9.x86_64 2024-05-09T03:35:48.649 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-05-09T03:35:48.649 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-05-09T03:35:48.649 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-05-09T03:35:48.649 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi132.front.sepia.ceph.com, path=None, version=distro) 2024-05-09T03:35:48.649 DEBUG:teuthology.orchestra.run.smithi132:> rpm -q kernel | sort -rV | head -n 1 2024-05-09T03:35:48.699 INFO:teuthology.orchestra.run.smithi132.stdout:kernel-5.14.0-444.el9.x86_64 2024-05-09T03:35:48.700 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-444.el9.x86_64 2024-05-09T03:35:48.700 DEBUG:teuthology.orchestra.run.smithi132:> sudo rpm -qi grub2-tools 2024-05-09T03:35:48.741 INFO:teuthology.orchestra.run.smithi132.stdout:Name : grub2-tools 2024-05-09T03:35:48.741 INFO:teuthology.orchestra.run.smithi132.stdout:Epoch : 1 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Version : 2.06 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Release : 68.el9 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Architecture: x86_64 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Group : Unspecified 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Size : 8264254 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:License : GPLv3+ 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Packager : builder@centos.org 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Vendor : CentOS 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-09T03:35:48.742 INFO:teuthology.orchestra.run.smithi132.stdout:Summary : Support tools for GRUB. 2024-05-09T03:35:48.743 INFO:teuthology.orchestra.run.smithi132.stdout:Description : 2024-05-09T03:35:48.743 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:35:48.743 INFO:teuthology.orchestra.run.smithi132.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-09T03:35:48.743 INFO:teuthology.orchestra.run.smithi132.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-09T03:35:48.743 INFO:teuthology.orchestra.run.smithi132.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-09T03:35:48.743 INFO:teuthology.orchestra.run.smithi132.stdout:hardware devices. 2024-05-09T03:35:48.743 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:35:48.743 INFO:teuthology.orchestra.run.smithi132.stdout:This subpackage provides tools for support of all platforms. 2024-05-09T03:35:48.744 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-09T03:35:48.744 INFO:teuthology.task.kernel:Updating grub on smithi132 to boot 5.14.0-444.el9.x86_64 2024-05-09T03:35:48.744 DEBUG:teuthology.orchestra.run.smithi132:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-09T03:35:49.110 INFO:teuthology.orchestra.run.smithi078.stdout:(3/4): kernel-modules-core-5.14.0-444.el9.x86_6 913 kB/s | 28 MB 00:31 2024-05-09T03:35:49.378 INFO:teuthology.orchestra.run.smithi132.stderr:Generating grub configuration file ... 2024-05-09T03:35:49.410 INFO:teuthology.orchestra.run.smithi078.stdout:(4/4): kernel-modules-5.14.0-444.el9.x86_64.rpm 1.0 MB/s | 34 MB 00:32 2024-05-09T03:35:49.410 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-05-09T03:35:49.410 INFO:teuthology.orchestra.run.smithi078.stdout:Total 2.4 MB/s | 78 MB 00:32 2024-05-09T03:35:50.094 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-05-09T03:35:50.426 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-05-09T03:35:50.426 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-05-09T03:35:50.791 INFO:teuthology.orchestra.run.smithi132.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-09T03:35:50.816 INFO:teuthology.orchestra.run.smithi132.stderr:done 2024-05-09T03:35:50.818 DEBUG:teuthology.orchestra.run.smithi132:> mktemp 2024-05-09T03:35:50.834 INFO:teuthology.orchestra.run.smithi132.stdout:/tmp/tmp.nlSEGxXE1G 2024-05-09T03:35:50.834 DEBUG:teuthology.orchestra.run.smithi132:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.nlSEGxXE1G 2024-05-09T03:35:50.913 DEBUG:teuthology.orchestra.run.smithi132:> sudo chmod 0666 /tmp/tmp.nlSEGxXE1G 2024-05-09T03:35:51.051 DEBUG:teuthology.orchestra.remote:smithi132:/tmp/tmp.nlSEGxXE1G is 6KB 2024-05-09T03:35:51.100 DEBUG:teuthology.orchestra.run.smithi132:> rm -fr /tmp/tmp.nlSEGxXE1G 2024-05-09T03:35:51.115 DEBUG:teuthology.orchestra.run.smithi132:> sudo /bin/ls /boot/loader/entries || true 2024-05-09T03:35:51.176 INFO:teuthology.orchestra.run.smithi132.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-09T03:35:51.176 INFO:teuthology.orchestra.run.smithi132.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-09T03:35:51.176 INFO:teuthology.orchestra.run.smithi132.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-09T03:35:51.176 INFO:teuthology.orchestra.run.smithi132.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-444.el9.x86_64.conf 2024-05-09T03:35:51.177 DEBUG:teuthology.orchestra.run.smithi132:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-444.el9.x86_64 2024-05-09T03:35:51.335 DEBUG:teuthology.orchestra.run.smithi132:> sudo shutdown -r now 2024-05-09T03:35:51.336 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-05-09T03:35:51.337 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-05-09T03:35:53.756 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-05-09T03:35:54.462 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-core-5.14.0-444.el9.x86_64 1/4 2024-05-09T03:35:54.536 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-core-5.14.0-444.el9.x86_64 2/4 2024-05-09T03:35:57.183 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-444.el9.x86_64 2/4 2024-05-09T03:35:57.318 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-5.14.0-444.el9.x86_64 3/4 2024-05-09T03:36:02.034 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-444.el9.x86_64 3/4 2024-05-09T03:36:02.079 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-5.14.0-444.el9.x86_64 4/4 2024-05-09T03:36:06.791 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-core-5.14.0-444.el9.x86_64 4/4 2024-05-09T03:36:21.338 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-05-09T03:36:21.339 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi132.front.sepia.ceph.com' 2024-05-09T03:36:21.340 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi132.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-09T03:36:36.136 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-444.el9.x86_64 4/4 2024-05-09T03:36:36.174 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-444.el9.x86_64 4/4 2024-05-09T03:36:36.453 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-5.14.0-444.el9.x86_64 4/4 2024-05-09T03:36:36.453 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-5.14.0-444.el9.x86_64 1/4 2024-05-09T03:36:36.453 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-core-5.14.0-444.el9.x86_64 2/4 2024-05-09T03:36:36.453 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-5.14.0-444.el9.x86_64 3/4 2024-05-09T03:36:37.218 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-core-5.14.0-444.el9.x86_64 4/4 2024-05-09T03:36:37.218 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:36:37.219 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-05-09T03:36:37.219 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-5.14.0-444.el9.x86_64 2024-05-09T03:36:37.219 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core-5.14.0-444.el9.x86_64 2024-05-09T03:36:37.219 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-5.14.0-444.el9.x86_64 2024-05-09T03:36:37.219 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core-5.14.0-444.el9.x86_64 2024-05-09T03:36:37.219 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:36:37.219 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-05-09T03:36:37.573 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-05-09T03:36:37.644 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-444.el9.x86_64 2024-05-09T03:36:37.644 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-444.el9.x86_64 2024-05-09T03:36:37.644 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-444.el9.x86_64 2024-05-09T03:36:37.644 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-05-09T03:36:37.644 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-05-09T03:36:37.644 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-05-09T03:36:37.644 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi078.front.sepia.ceph.com, path=None, version=distro) 2024-05-09T03:36:37.645 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-05-09T03:36:37.695 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-444.el9.x86_64 2024-05-09T03:36:37.695 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-444.el9.x86_64 2024-05-09T03:36:37.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo rpm -qi grub2-tools 2024-05-09T03:36:37.727 INFO:teuthology.orchestra.run.smithi078.stdout:Name : grub2-tools 2024-05-09T03:36:37.727 INFO:teuthology.orchestra.run.smithi078.stdout:Epoch : 1 2024-05-09T03:36:37.727 INFO:teuthology.orchestra.run.smithi078.stdout:Version : 2.06 2024-05-09T03:36:37.728 INFO:teuthology.orchestra.run.smithi078.stdout:Release : 68.el9 2024-05-09T03:36:37.728 INFO:teuthology.orchestra.run.smithi078.stdout:Architecture: x86_64 2024-05-09T03:36:37.728 INFO:teuthology.orchestra.run.smithi078.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-09T03:36:37.728 INFO:teuthology.orchestra.run.smithi078.stdout:Group : Unspecified 2024-05-09T03:36:37.728 INFO:teuthology.orchestra.run.smithi078.stdout:Size : 8264254 2024-05-09T03:36:37.728 INFO:teuthology.orchestra.run.smithi078.stdout:License : GPLv3+ 2024-05-09T03:36:37.729 INFO:teuthology.orchestra.run.smithi078.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-09T03:36:37.729 INFO:teuthology.orchestra.run.smithi078.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-09T03:36:37.729 INFO:teuthology.orchestra.run.smithi078.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-09T03:36:37.729 INFO:teuthology.orchestra.run.smithi078.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-09T03:36:37.729 INFO:teuthology.orchestra.run.smithi078.stdout:Packager : builder@centos.org 2024-05-09T03:36:37.729 INFO:teuthology.orchestra.run.smithi078.stdout:Vendor : CentOS 2024-05-09T03:36:37.730 INFO:teuthology.orchestra.run.smithi078.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-09T03:36:37.730 INFO:teuthology.orchestra.run.smithi078.stdout:Summary : Support tools for GRUB. 2024-05-09T03:36:37.730 INFO:teuthology.orchestra.run.smithi078.stdout:Description : 2024-05-09T03:36:37.730 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:36:37.730 INFO:teuthology.orchestra.run.smithi078.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-09T03:36:37.730 INFO:teuthology.orchestra.run.smithi078.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-09T03:36:37.731 INFO:teuthology.orchestra.run.smithi078.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-09T03:36:37.731 INFO:teuthology.orchestra.run.smithi078.stdout:hardware devices. 2024-05-09T03:36:37.731 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:36:37.731 INFO:teuthology.orchestra.run.smithi078.stdout:This subpackage provides tools for support of all platforms. 2024-05-09T03:36:37.732 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-09T03:36:37.733 INFO:teuthology.task.kernel:Updating grub on smithi078 to boot 5.14.0-444.el9.x86_64 2024-05-09T03:36:37.733 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-09T03:36:38.418 INFO:teuthology.orchestra.run.smithi078.stderr:Generating grub configuration file ... 2024-05-09T03:36:39.710 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.132 2024-05-09T03:36:39.741 INFO:teuthology.orchestra.run.smithi078.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-09T03:36:39.761 INFO:teuthology.orchestra.run.smithi078.stderr:done 2024-05-09T03:36:39.764 DEBUG:teuthology.orchestra.run.smithi078:> mktemp 2024-05-09T03:36:39.789 INFO:teuthology.orchestra.run.smithi078.stdout:/tmp/tmp.RxcuEvFFYk 2024-05-09T03:36:39.789 DEBUG:teuthology.orchestra.run.smithi078:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.RxcuEvFFYk 2024-05-09T03:36:39.859 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0666 /tmp/tmp.RxcuEvFFYk 2024-05-09T03:36:39.999 DEBUG:teuthology.orchestra.remote:smithi078:/tmp/tmp.RxcuEvFFYk is 6KB 2024-05-09T03:36:40.049 DEBUG:teuthology.orchestra.run.smithi078:> rm -fr /tmp/tmp.RxcuEvFFYk 2024-05-09T03:36:40.063 DEBUG:teuthology.orchestra.run.smithi078:> sudo /bin/ls /boot/loader/entries || true 2024-05-09T03:36:40.125 INFO:teuthology.orchestra.run.smithi078.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-09T03:36:40.125 INFO:teuthology.orchestra.run.smithi078.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-09T03:36:40.125 INFO:teuthology.orchestra.run.smithi078.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-09T03:36:40.125 INFO:teuthology.orchestra.run.smithi078.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-444.el9.x86_64.conf 2024-05-09T03:36:40.127 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-444.el9.x86_64 2024-05-09T03:36:40.268 DEBUG:teuthology.orchestra.run.smithi078:> sudo shutdown -r now 2024-05-09T03:36:48.718 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi132.front.sepia.ceph.com' 2024-05-09T03:36:48.719 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi132.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-09T03:37:10.290 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-05-09T03:37:10.291 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-09T03:37:10.291 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-09T03:37:28.607 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-05-09T03:37:37.612 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-09T03:37:37.613 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-09T03:37:40.674 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-05-09T03:37:48.722 DEBUG:teuthology.orchestra.remote:timed out 2024-05-09T03:37:52.678 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-09T03:37:52.679 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-09T03:38:00.730 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi132.front.sepia.ceph.com' 2024-05-09T03:38:00.731 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi132.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-09T03:38:00.995 DEBUG:teuthology.orchestra.run.smithi132:> true 2024-05-09T03:38:01.346 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi132.front.sepia.ceph.com' 2024-05-09T03:38:01.346 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-444.el9.x86_64"... 2024-05-09T03:38:01.346 DEBUG:teuthology.orchestra.run.smithi132:> uname -r 2024-05-09T03:38:01.363 INFO:teuthology.orchestra.run.smithi132.stdout:5.14.0-444.el9.x86_64 2024-05-09T03:38:01.363 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-444.el9.x86_64 vs 5.14.0-444.el9.x86_64 2024-05-09T03:38:01.363 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-09T03:38:01.363 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-09T03:38:02.364 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-05-09T03:38:02.364 DEBUG:teuthology.orchestra.run.smithi132:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-09T03:38:02.437 INFO:teuthology.orchestra.run.smithi132.stdout:ttyS1 2024-05-09T03:38:02.472 DEBUG:teuthology.parallel:result is None 2024-05-09T03:38:11.102 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-05-09T03:38:26.103 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-09T03:38:26.104 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-09T03:38:26.418 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-05-09T03:38:26.858 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-09T03:38:26.859 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-444.el9.x86_64"... 2024-05-09T03:38:26.859 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-05-09T03:38:26.873 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-444.el9.x86_64 2024-05-09T03:38:26.874 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-444.el9.x86_64 vs 5.14.0-444.el9.x86_64 2024-05-09T03:38:26.874 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-09T03:38:26.874 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-09T03:38:27.874 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-05-09T03:38:27.875 DEBUG:teuthology.orchestra.run.smithi078:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-09T03:38:27.953 INFO:teuthology.orchestra.run.smithi078.stdout:ttyS1 2024-05-09T03:38:27.984 DEBUG:teuthology.parallel:result is None 2024-05-09T03:38:27.984 INFO:teuthology.run_tasks:Running task internal.base... 2024-05-09T03:38:27.996 INFO:teuthology.task.internal:Creating test directory... 2024-05-09T03:38:27.996 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-09T03:38:27.998 DEBUG:teuthology.orchestra.run.smithi132:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-09T03:38:28.017 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-05-09T03:38:28.029 INFO:teuthology.run_tasks:Running task internal.archive... 2024-05-09T03:38:28.042 INFO:teuthology.task.internal:Creating archive directory... 2024-05-09T03:38:28.042 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-09T03:38:28.053 DEBUG:teuthology.orchestra.run.smithi132:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-09T03:38:28.173 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-05-09T03:38:28.189 INFO:teuthology.task.internal:Enabling coredump saving... 2024-05-09T03:38:28.189 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-09T03:38:28.191 DEBUG:teuthology.orchestra.run.smithi132:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-09T03:38:28.226 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-09T03:38:28.237 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-09T03:38:28.240 INFO:teuthology.orchestra.run.smithi132.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-09T03:38:28.252 INFO:teuthology.orchestra.run.smithi132.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-09T03:38:28.254 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-05-09T03:38:28.265 INFO:teuthology.task.internal:Configuring sudo... 2024-05-09T03:38:28.265 DEBUG:teuthology.orchestra.run.smithi078:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-09T03:38:28.281 DEBUG:teuthology.orchestra.run.smithi132:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-09T03:38:28.309 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-05-09T03:38:28.320 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-05-09T03:38:28.320 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-09T03:38:28.347 DEBUG:teuthology.orchestra.run.smithi132:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-09T03:38:28.367 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-09T03:38:28.444 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-09T03:38:28.491 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-09T03:38:28.577 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-09T03:38:28.615 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-09T03:38:28.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-09T03:38:28.681 DEBUG:teuthology.orchestra.run.smithi132:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-09T03:38:28.729 DEBUG:teuthology.orchestra.run.smithi132:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-09T03:38:28.774 DEBUG:teuthology.orchestra.run.smithi132:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-09T03:38:28.856 DEBUG:teuthology.orchestra.run.smithi132:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-09T03:38:28.902 DEBUG:teuthology.orchestra.run.smithi132:> set -ex 2024-05-09T03:38:28.902 DEBUG:teuthology.orchestra.run.smithi132:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-09T03:38:28.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo service rsyslog restart 2024-05-09T03:38:28.971 DEBUG:teuthology.orchestra.run.smithi132:> sudo service rsyslog restart 2024-05-09T03:38:29.011 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-09T03:38:29.079 INFO:teuthology.orchestra.run.smithi132.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-09T03:38:29.541 INFO:teuthology.run_tasks:Running task internal.timer... 2024-05-09T03:38:29.550 INFO:teuthology.task.internal:Starting timer... 2024-05-09T03:38:29.550 INFO:teuthology.run_tasks:Running task pcp... 2024-05-09T03:38:29.575 INFO:teuthology.run_tasks:Running task selinux... 2024-05-09T03:38:29.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo service auditd rotate 2024-05-09T03:38:29.630 INFO:teuthology.orchestra.run.smithi078.stdout:Rotating logs: 2024-05-09T03:38:29.632 DEBUG:teuthology.orchestra.run.smithi132:> sudo service auditd rotate 2024-05-09T03:38:29.711 INFO:teuthology.orchestra.run.smithi132.stdout:Rotating logs: 2024-05-09T03:38:29.713 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-05-09T03:38:29.713 DEBUG:teuthology.orchestra.run.smithi078:> /usr/sbin/getenforce 2024-05-09T03:38:29.737 INFO:teuthology.orchestra.run.smithi078.stdout:Permissive 2024-05-09T03:38:29.737 DEBUG:teuthology.orchestra.run.smithi132:> /usr/sbin/getenforce 2024-05-09T03:38:29.769 INFO:teuthology.orchestra.run.smithi132.stdout:Permissive 2024-05-09T03:38:29.770 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi078.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi132.front.sepia.ceph.com': 'permissive'} 2024-05-09T03:38:29.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-09T03:38:29.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-09T03:38:29.803 DEBUG:teuthology.orchestra.run.smithi132:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-09T03:38:29.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-09T03:38:29.837 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-05-09T03:38:29.837 DEBUG:teuthology.orchestra.run.smithi078:> sudo /usr/sbin/setenforce permissive 2024-05-09T03:38:29.987 DEBUG:teuthology.orchestra.run.smithi132:> sudo /usr/sbin/setenforce permissive 2024-05-09T03:38:30.015 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-05-09T03:38:30.026 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-05-09T03:38:30.043 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-05-09T03:38:30.044 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi078.front.sepia.ceph.com,smithi132.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-05-09T03:42:26.285 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi078.front.sepia.ceph.com'), Remote(name='ubuntu@smithi132.front.sepia.ceph.com')] 2024-05-09T03:42:26.286 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-09T03:42:26.288 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-09T03:42:26.365 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-05-09T03:42:26.446 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-09T03:42:26.446 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi132.front.sepia.ceph.com' 2024-05-09T03:42:26.447 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi132.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-09T03:42:26.525 DEBUG:teuthology.orchestra.run.smithi132:> true 2024-05-09T03:42:26.606 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi132.front.sepia.ceph.com' 2024-05-09T03:42:26.607 INFO:teuthology.run_tasks:Running task clock... 2024-05-09T03:42:26.617 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-05-09T03:42:26.618 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-09T03:42:26.618 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-09T03:42:26.620 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-09T03:42:26.621 DEBUG:teuthology.orchestra.run.smithi132:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-09T03:42:26.647 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-09T03:42:26.661 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-09T03:42:26.677 INFO:teuthology.orchestra.run.smithi132.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-09T03:42:26.686 INFO:teuthology.orchestra.run.smithi078.stderr:sudo: ntpd: command not found 2024-05-09T03:42:26.691 INFO:teuthology.orchestra.run.smithi132.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-09T03:42:26.697 INFO:teuthology.orchestra.run.smithi078.stdout:506 Cannot talk to daemon 2024-05-09T03:42:26.710 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-09T03:42:26.716 INFO:teuthology.orchestra.run.smithi132.stderr:sudo: ntpd: command not found 2024-05-09T03:42:26.724 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-09T03:42:26.726 INFO:teuthology.orchestra.run.smithi132.stdout:506 Cannot talk to daemon 2024-05-09T03:42:26.740 INFO:teuthology.orchestra.run.smithi132.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-09T03:42:26.754 INFO:teuthology.orchestra.run.smithi132.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-09T03:42:26.778 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-05-09T03:42:26.782 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-09T03:42:26.782 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-05-09T03:42:26.782 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-09T03:42:26.783 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-09T03:42:26.783 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-09T03:42:26.783 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-09T03:42:26.809 INFO:teuthology.orchestra.run.smithi132.stderr:bash: line 1: ntpq: command not found 2024-05-09T03:42:26.812 INFO:teuthology.orchestra.run.smithi132.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-09T03:42:26.812 INFO:teuthology.orchestra.run.smithi132.stdout:=============================================================================== 2024-05-09T03:42:26.812 INFO:teuthology.orchestra.run.smithi132.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-09T03:42:26.812 INFO:teuthology.orchestra.run.smithi132.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-09T03:42:26.813 INFO:teuthology.orchestra.run.smithi132.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-09T03:42:26.813 INFO:teuthology.orchestra.run.smithi132.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-09T03:42:26.814 INFO:teuthology.run_tasks:Running task cephadm... 2024-05-09T03:42:26.913 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_STRAY_DAEMON'], 'log-only-match': ['CEPHADM_'], 'sha1': 'c66c553a84eb9faa0029f5f0bb8284e3a826ac85'} 2024-05-09T03:42:26.914 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-05-09T03:42:26.914 INFO:tasks.cephadm:Cluster fsid is 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:42:26.914 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-05-09T03:42:26.918 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.78', 'mon.c': '[v2:172.21.15.78:3301,v1:172.21.15.78:6790]', 'mon.b': '172.21.15.132'} 2024-05-09T03:42:26.918 INFO:tasks.cephadm:First mon is mon.a on smithi078 2024-05-09T03:42:26.918 INFO:tasks.cephadm:First mgr is y 2024-05-09T03:42:26.918 INFO:tasks.cephadm:Normalizing hostnames... 2024-05-09T03:42:26.918 DEBUG:teuthology.orchestra.run.smithi078:> sudo hostname $(hostname -s) 2024-05-09T03:42:26.945 DEBUG:teuthology.orchestra.run.smithi132:> sudo hostname $(hostname -s) 2024-05-09T03:42:26.970 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-05-09T03:42:26.970 DEBUG:teuthology.orchestra.run.smithi078:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-09T03:42:27.212 INFO:teuthology.orchestra.run.smithi078.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 9 03:42 /home/ubuntu/cephtest/cephadm 2024-05-09T03:42:27.213 DEBUG:teuthology.orchestra.run.smithi132:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-09T03:42:27.381 INFO:teuthology.orchestra.run.smithi132.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 9 03:42 /home/ubuntu/cephtest/cephadm 2024-05-09T03:42:27.382 DEBUG:teuthology.orchestra.run.smithi078:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-09T03:42:27.398 DEBUG:teuthology.orchestra.run.smithi132:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-09T03:42:27.424 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-05-09T03:42:27.424 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-09T03:42:27.441 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-09T03:42:29.052 INFO:teuthology.orchestra.run.smithi132.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-09T03:42:29.136 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-09T03:42:48.100 INFO:teuthology.orchestra.run.smithi132.stdout:{ 2024-05-09T03:42:48.100 INFO:teuthology.orchestra.run.smithi132.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-09T03:42:48.100 INFO:teuthology.orchestra.run.smithi132.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-09T03:42:48.100 INFO:teuthology.orchestra.run.smithi132.stdout: "repo_digests": [ 2024-05-09T03:42:48.100 INFO:teuthology.orchestra.run.smithi132.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-09T03:42:48.100 INFO:teuthology.orchestra.run.smithi132.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-09T03:42:48.100 INFO:teuthology.orchestra.run.smithi132.stdout: ] 2024-05-09T03:42:48.101 INFO:teuthology.orchestra.run.smithi132.stdout:} 2024-05-09T03:42:49.212 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T03:42:49.212 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-09T03:42:49.212 INFO:teuthology.orchestra.run.smithi078.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-09T03:42:49.212 INFO:teuthology.orchestra.run.smithi078.stdout: "repo_digests": [ 2024-05-09T03:42:49.212 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-09T03:42:49.212 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-09T03:42:49.212 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-05-09T03:42:49.212 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T03:42:49.230 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /etc/ceph 2024-05-09T03:42:49.289 DEBUG:teuthology.orchestra.run.smithi132:> sudo mkdir -p /etc/ceph 2024-05-09T03:42:49.321 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 777 /etc/ceph 2024-05-09T03:42:49.346 DEBUG:teuthology.orchestra.run.smithi132:> sudo chmod 777 /etc/ceph 2024-05-09T03:42:49.389 INFO:tasks.cephadm:Writing seed config... 2024-05-09T03:42:49.390 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-05-09T03:42:49.391 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-05-09T03:42:49.391 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-05-09T03:42:49.391 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-05-09T03:42:49.391 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-05-09T03:42:49.391 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-05-09T03:42:49.391 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-05-09T03:42:49.391 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-05-09T03:42:49.391 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-05-09T03:42:49.392 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-09T03:42:49.392 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-05-09T03:42:49.407 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 2777c79e-0db6-11ef-bc97-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-05-09T03:42:49.407 DEBUG:teuthology.orchestra.run.smithi078:mon.a> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.a.service 2024-05-09T03:42:49.450 DEBUG:teuthology.orchestra.run.smithi078:mgr.y> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.y.service 2024-05-09T03:42:49.492 INFO:tasks.cephadm:Bootstrapping... 2024-05-09T03:42:49.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.78 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-05-09T03:42:49.684 INFO:teuthology.orchestra.run.smithi078.stderr:-------------------------------------------------------------------------------- 2024-05-09T03:42:49.685 INFO:teuthology.orchestra.run.smithi078.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '2777c79e-0db6-11ef-bc97-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.78', '--skip-admin-label'] 2024-05-09T03:42:49.708 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 4.9.4-dev 2024-05-09T03:42:49.710 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying podman|docker is present... 2024-05-09T03:42:49.732 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 4.9.4-dev 2024-05-09T03:42:49.734 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying lvm2 is present... 2024-05-09T03:42:49.734 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying time synchronization is in place... 2024-05-09T03:42:49.741 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-09T03:42:49.747 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-05-09T03:42:49.755 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: enabled 2024-05-09T03:42:49.761 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: active 2024-05-09T03:42:49.762 INFO:teuthology.orchestra.run.smithi078.stderr:Unit chronyd.service is enabled and running 2024-05-09T03:42:49.762 INFO:teuthology.orchestra.run.smithi078.stderr:Repeating the final host check... 2024-05-09T03:42:49.783 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 4.9.4-dev 2024-05-09T03:42:49.785 INFO:teuthology.orchestra.run.smithi078.stderr:podman (/bin/podman) version 4.9.4 is present 2024-05-09T03:42:49.785 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl is present 2024-05-09T03:42:49.785 INFO:teuthology.orchestra.run.smithi078.stderr:lvcreate is present 2024-05-09T03:42:49.791 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-09T03:42:49.798 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-05-09T03:42:49.804 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: enabled 2024-05-09T03:42:49.810 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: active 2024-05-09T03:42:49.811 INFO:teuthology.orchestra.run.smithi078.stderr:Unit chronyd.service is enabled and running 2024-05-09T03:42:49.811 INFO:teuthology.orchestra.run.smithi078.stderr:Host looks OK 2024-05-09T03:42:49.811 INFO:teuthology.orchestra.run.smithi078.stderr:Cluster fsid: 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:42:49.812 INFO:teuthology.orchestra.run.smithi078.stderr:Acquiring lock 140485575868272 on /run/cephadm/2777c79e-0db6-11ef-bc97-c7b262605968.lock 2024-05-09T03:42:49.812 INFO:teuthology.orchestra.run.smithi078.stderr:Lock 140485575868272 acquired on /run/cephadm/2777c79e-0db6-11ef-bc97-c7b262605968.lock 2024-05-09T03:42:49.812 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying IP 172.21.15.78 port 3300 ... 2024-05-09T03:42:49.812 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying IP 172.21.15.78 port 6789 ... 2024-05-09T03:42:49.812 INFO:teuthology.orchestra.run.smithi078.stderr:Base mon IP is 172.21.15.78, final addrv is [v2:172.21.15.78:3300,v1:172.21.15.78:6789] 2024-05-09T03:42:49.815 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.78 metric 100 2024-05-09T03:42:49.815 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.78 metric 100 2024-05-09T03:42:49.817 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-05-09T03:42:49.818 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-05-09T03:42:49.818 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1771sec hoplimit 64 pref medium 2024-05-09T03:42:49.820 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-05-09T03:42:49.820 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-05-09T03:42:49.820 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-09T03:42:49.820 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-05-09T03:42:49.821 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d4dd/64 scope link 2024-05-09T03:42:49.821 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-09T03:42:49.821 INFO:teuthology.orchestra.run.smithi078.stderr:Mon IP `172.21.15.78` is in CIDR network `172.21.0.0/20` 2024-05-09T03:42:49.821 INFO:teuthology.orchestra.run.smithi078.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-05-09T03:42:49.822 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-09T03:42:49.924 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-05-09T03:42:50.276 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Getting image source signatures 2024-05-09T03:42:50.276 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-05-09T03:42:50.276 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-05-09T03:42:50.276 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-05-09T03:42:50.276 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-05-09T03:42:50.276 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-05-09T03:42:50.276 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-09T03:42:50.433 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Writing manifest to image destination 2024-05-09T03:42:50.627 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-09T03:42:50.996 INFO:teuthology.orchestra.run.smithi078.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-09T03:42:51.535 INFO:teuthology.orchestra.run.smithi078.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-09T03:42:51.535 INFO:teuthology.orchestra.run.smithi078.stderr:Extracting ceph user uid/gid from container image... 2024-05-09T03:42:51.926 INFO:teuthology.orchestra.run.smithi078.stderr:stat: 167 167 2024-05-09T03:42:52.518 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial keys... 2024-05-09T03:42:52.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQA8Rjxm8hiHMhAAWgaAUNTGfZSo7b14mj2ElA== 2024-05-09T03:42:53.887 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQA9RjxmltjkLxAAiXIrtLR9DDzfp2EbL3Il9Q== 2024-05-09T03:42:54.793 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQA+Rjxm0ubRKRAA+YIqO74IqKx33E0Wk+tOCQ== 2024-05-09T03:42:55.283 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial monmap... 2024-05-09T03:42:55.708 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-09T03:42:55.708 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-05-09T03:42:55.708 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:42:55.708 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-09T03:42:56.281 INFO:teuthology.orchestra.run.smithi078.stderr:monmaptool for a [v2:172.21.15.78:3300,v1:172.21.15.78:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-09T03:42:56.282 INFO:teuthology.orchestra.run.smithi078.stderr:setting min_mon_release = octopus 2024-05-09T03:42:56.282 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: set fsid to 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:42:56.282 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-09T03:42:56.282 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-05-09T03:42:56.282 INFO:teuthology.orchestra.run.smithi078.stderr:Creating mon... 2024-05-09T03:42:57.205 INFO:teuthology.orchestra.run.smithi078.stderr:create mon.a on 2024-05-09T03:42:57.409 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-05-09T03:42:57.559 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-2777c79e-0db6-11ef-bc97-c7b262605968.target → /etc/systemd/system/ceph-2777c79e-0db6-11ef-bc97-c7b262605968.target. 2024-05-09T03:42:57.559 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-2777c79e-0db6-11ef-bc97-c7b262605968.target → /etc/systemd/system/ceph-2777c79e-0db6-11ef-bc97-c7b262605968.target. 2024-05-09T03:42:57.852 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to reset failed state of unit ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.a.service: Unit ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.a.service not loaded. 2024-05-09T03:42:57.860 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph-2777c79e-0db6-11ef-bc97-c7b262605968.target.wants/ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.a.service → /etc/systemd/system/ceph-2777c79e-0db6-11ef-bc97-c7b262605968@.service. 2024-05-09T03:42:58.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:42:58 smithi078 podman[22544]: 2024-05-09 03:42:58.475911983 +0000 UTC m=+0.328456235 container start 162bc9c32b908f99cf490c82e753f7a69312a8024799635063e9dde4bf125e69 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_CLEAN=True, version=8, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.component=centos-stream-container) 2024-05-09T03:42:58.561 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-05-09T03:42:58.568 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-05-09T03:42:58.568 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-05-09T03:42:58.568 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to enable service . firewalld.service is not available 2024-05-09T03:42:58.569 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mon to start... 2024-05-09T03:42:58.569 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mon... 2024-05-09T03:42:58.786 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:42:58 smithi078 ceph-mon[22563]: mkfs 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:42:58.786 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:42:58 smithi078 ceph-mon[22563]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-09T03:42:58.786 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:42:58 smithi078 bash[22544]: 162bc9c32b908f99cf490c82e753f7a69312a8024799635063e9dde4bf125e69 2024-05-09T03:42:58.786 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:42:58 smithi078 systemd[1]: Started Ceph mon.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:42:59.046 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: cluster: 2024-05-09T03:42:59.046 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: id: 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:42:59.046 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-05-09T03:42:59.046 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-05-09T03:42:59.046 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: services: 2024-05-09T03:42:59.046 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.531123s) 2024-05-09T03:42:59.046 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr: no daemons active 2024-05-09T03:42:59.046 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-05-09T03:42:59.046 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-05-09T03:42:59.047 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: data: 2024-05-09T03:42:59.047 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-05-09T03:42:59.047 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-05-09T03:42:59.047 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-05-09T03:42:59.047 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: pgs: 2024-05-09T03:42:59.047 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-05-09T03:42:59.557 INFO:teuthology.orchestra.run.smithi078.stderr:mon is available 2024-05-09T03:42:59.558 INFO:teuthology.orchestra.run.smithi078.stderr:Assimilating anything we can from ceph.conf... 2024-05-09T03:42:59.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:42:59 smithi078 ceph-mon[22563]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-09T03:42:59.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:42:59 smithi078 ceph-mon[22563]: monmap e1: 1 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} 2024-05-09T03:42:59.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:42:59 smithi078 ceph-mon[22563]: fsmap 2024-05-09T03:42:59.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:42:59 smithi078 ceph-mon[22563]: osdmap e1: 0 total, 0 up, 0 in 2024-05-09T03:42:59.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:42:59 smithi078 ceph-mon[22563]: mgrmap e1: no daemons active 2024-05-09T03:42:59.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:42:59 smithi078 ceph-mon[22563]: from='client.? 172.21.15.78:0/1390309606' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T03:43:00.062 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-05-09T03:43:00.062 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [global] 2024-05-09T03:43:00.062 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: fsid = 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:43:00.062 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.78:3300,v1:172.21.15.78:6789] 2024-05-09T03:43:00.062 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-09T03:43:00.063 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-09T03:43:00.063 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-09T03:43:00.063 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-09T03:43:00.063 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-05-09T03:43:00.063 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [mgr] 2024-05-09T03:43:00.063 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-05-09T03:43:00.063 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-09T03:43:00.063 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-05-09T03:43:00.063 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [osd] 2024-05-09T03:43:00.063 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-09T03:43:00.063 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-09T03:43:00.676 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:00 smithi078 ceph-mon[22563]: from='client.? 172.21.15.78:0/1740884200' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-09T03:43:00.676 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:00 smithi078 ceph-mon[22563]: from='client.? 172.21.15.78:0/1740884200' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-09T03:43:00.678 INFO:teuthology.orchestra.run.smithi078.stderr:Generating new minimal ceph.conf... 2024-05-09T03:43:01.676 INFO:teuthology.orchestra.run.smithi078.stderr:Restarting the monitor... 2024-05-09T03:43:01.720 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:01 smithi078 ceph-mon[22563]: from='client.? 172.21.15.78:0/3349714608' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:43:01.720 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:01 smithi078 systemd[1]: Stopping Ceph mon.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:43:02.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:01 smithi078 bash[22782]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon.a" found: no such container 2024-05-09T03:43:02.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:01 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a[22559]: 2024-05-09T03:43:01.851+0000 7f10605b8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T03:43:02.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:01 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a[22559]: 2024-05-09T03:43:01.851+0000 7f10605b8700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-05-09T03:43:02.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:01 smithi078 podman[22793]: 2024-05-09 03:43:01.954254586 +0000 UTC m=+0.224772153 container died 162bc9c32b908f99cf490c82e753f7a69312a8024799635063e9dde4bf125e69 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, maintainer=Guillaume Abrioux , version=8, io.buildah.version=1.19.8, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, architecture=x86_64, release=754, build-date=2022-05-03T08:36:31.336870, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-05-09T03:43:02.460 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 podman[22793]: 2024-05-09 03:43:02.268250742 +0000 UTC m=+0.538768354 container cleanup 162bc9c32b908f99cf490c82e753f7a69312a8024799635063e9dde4bf125e69 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, name=centos-stream, GIT_BRANCH=HEAD, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vendor=Red Hat, Inc., version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD) 2024-05-09T03:43:02.460 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 bash[22793]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a 2024-05-09T03:43:02.460 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 bash[22822]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon.a" found: no such container 2024-05-09T03:43:02.460 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 podman[22807]: 2024-05-09 03:43:02.435585954 +0000 UTC m=+0.484654115 container remove 162bc9c32b908f99cf490c82e753f7a69312a8024799635063e9dde4bf125e69 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , release=754, GIT_BRANCH=HEAD, version=8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, vendor=Red Hat, Inc., architecture=x86_64, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, name=centos-stream, GIT_CLEAN=True, com.redhat.component=centos-stream-container) 2024-05-09T03:43:02.460 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.a.service: Deactivated successfully. 2024-05-09T03:43:02.460 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 systemd[1]: Stopped Ceph mon.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:43:02.461 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 systemd[1]: Starting Ceph mon.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:43:02.883 INFO:teuthology.orchestra.run.smithi078.stderr:Setting mon public_network to 172.21.0.0/20 2024-05-09T03:43:03.006 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 podman[22887]: 2024-05-09 03:43:02.57853051 +0000 UTC m=+0.018743870 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-05-09T03:43:03.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 podman[22887]: 2024-05-09 03:43:02.678108186 +0000 UTC m=+0.118321551 container create de97a71c7cb50369d585b52c272637b11e40f5dd85f89fb697ce1dcef897d9ee (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.buildah.version=1.19.8, distribution-scope=public, ceph=True, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, com.redhat.component=centos-stream-container, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, version=8, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream) 2024-05-09T03:43:03.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 podman[22887]: 2024-05-09 03:43:02.830268857 +0000 UTC m=+0.270482220 container init de97a71c7cb50369d585b52c272637b11e40f5dd85f89fb697ce1dcef897d9ee (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, ceph=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, name=centos-stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, vcs-type=git, version=8, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., RELEASE=HEAD, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64) 2024-05-09T03:43:03.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 podman[22887]: 2024-05-09 03:43:02.833293727 +0000 UTC m=+0.273507088 container start de97a71c7cb50369d585b52c272637b11e40f5dd85f89fb697ce1dcef897d9ee (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, distribution-scope=public, name=centos-stream, vcs-type=git, io.openshift.expose-services=, version=8, ceph=True, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.buildah.version=1.19.8) 2024-05-09T03:43:03.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: set uid:gid to 167:167 (ceph:ceph) 2024-05-09T03:43:03.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-09T03:43:03.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: pidfile_write: ignore empty --pid-file 2024-05-09T03:43:03.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: load: jerasure load: lrc 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: RocksDB version: 6.15.5 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Compile date Apr 18 2022 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: DB SUMMARY 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: DB Session ID: NE8VUQ9S4MFHNQMH420G 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: CURRENT file: CURRENT 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: IDENTITY file: IDENTITY 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82637 ; 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.error_if_exists: 0 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.create_if_missing: 0 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.paranoid_checks: 1 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-09T03:43:03.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.env: 0x56060d2c6860 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.fs: Posix File System 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.info_log: 0x56060f587dc0 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_file_opening_threads: 16 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.statistics: (nil) 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.use_fsync: 0 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_log_file_size: 0 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.keep_log_file_num: 1000 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.recycle_log_file_num: 0 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.allow_fallocate: 1 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.allow_mmap_reads: 0 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.allow_mmap_writes: 0 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.use_direct_reads: 0 2024-05-09T03:43:03.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.create_missing_column_families: 0 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.db_log_dir: 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.advise_random_on_open: 1 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.db_write_buffer_size: 0 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.write_buffer_manager: 0x56060f6782a0 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-09T03:43:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.rate_limiter: (nil) 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.wal_recovery_mode: 2 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.enable_thread_tracking: 0 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.enable_pipelined_write: 0 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.unordered_write: 0 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.row_cache: None 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.wal_filter: None 2024-05-09T03:43:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-09T03:43:03.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.allow_ingest_behind: 0 2024-05-09T03:43:03.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.preserve_deletes: 0 2024-05-09T03:43:03.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.two_write_queues: 0 2024-05-09T03:43:03.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.manual_wal_flush: 0 2024-05-09T03:43:03.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.atomic_flush: 0 2024-05-09T03:43:03.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-09T03:43:03.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-09T03:43:03.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-09T03:43:03.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.log_readahead_size: 0 2024-05-09T03:43:03.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-09T03:43:03.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.best_efforts_recovery: 0 2024-05-09T03:43:03.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.allow_data_in_errors: 0 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.db_host_id: __hostname__ 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_background_jobs: 2 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_background_compactions: -1 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_subcompactions: 1 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_total_wal_size: 0 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-09T03:43:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_open_files: -1 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.bytes_per_sync: 0 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_readahead_size: 0 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_background_flushes: -1 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Compression algorithms supported: 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: kZSTD supported: 0 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: kXpressCompression supported: 0 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: kLZ4HCCompression supported: 1 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: kLZ4Compression supported: 1 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: kBZip2Compression supported: 0 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: kZlibCompression supported: 1 2024-05-09T03:43:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: kSnappyCompression supported: 1 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.merge_operator: 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_filter: None 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_filter_factory: None 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.sst_partitioner_factory: None 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56060f553d00) 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-09T03:43:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: index_type: 0 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: data_block_index_type: 0 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: index_shortening: 1 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: hash_index_allow_collision: 1 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: checksum: 1 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: no_block_cache: 0 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_cache: 0x56060f5be170 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_cache_options: 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: capacity : 536870912 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: num_shard_bits : 4 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: strict_capacity_limit : 0 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_cache_compressed: (nil) 2024-05-09T03:43:03.017 INFO:journalctl@ceph.mon.a.smithi078.stdout: persistent_cache: (nil) 2024-05-09T03:43:03.018 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_size: 4096 2024-05-09T03:43:03.018 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_size_deviation: 10 2024-05-09T03:43:03.018 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_restart_interval: 16 2024-05-09T03:43:03.018 INFO:journalctl@ceph.mon.a.smithi078.stdout: index_block_restart_interval: 1 2024-05-09T03:43:03.018 INFO:journalctl@ceph.mon.a.smithi078.stdout: metadata_block_size: 4096 2024-05-09T03:43:03.018 INFO:journalctl@ceph.mon.a.smithi078.stdout: partition_filters: 0 2024-05-09T03:43:03.018 INFO:journalctl@ceph.mon.a.smithi078.stdout: use_delta_encoding: 1 2024-05-09T03:43:03.018 INFO:journalctl@ceph.mon.a.smithi078.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-09T03:43:03.018 INFO:journalctl@ceph.mon.a.smithi078.stdout: whole_key_filtering: 1 2024-05-09T03:43:03.018 INFO:journalctl@ceph.mon.a.smithi078.stdout: verify_compression: 0 2024-05-09T03:43:03.018 INFO:journalctl@ceph.mon.a.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout: format_version: 4 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout: enable_index_compression: 1 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_align: 0 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.write_buffer_size: 33554432 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_write_buffer_number: 2 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compression: NoCompression 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.bottommost_compression: Disabled 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.prefix_extractor: nullptr 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.num_levels: 7 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-09T03:43:03.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compression_opts.level: 32767 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compression_opts.strategy: 0 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compression_opts.enabled: false 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-09T03:43:03.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.target_file_size_base: 67108864 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-09T03:43:03.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.arena_block_size: 4194304 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.disable_auto_compactions: 0 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-09T03:43:03.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.table_properties_collectors: 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.inplace_update_support: 0 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.bloom_locality: 0 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.max_successive_merges: 0 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.paranoid_file_checks: 0 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.force_consistency_checks: 1 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.report_bg_io_stats: 0 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.ttl: 2592000 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.enable_blob_files: false 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.min_blob_size: 0 2024-05-09T03:43:03.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.blob_file_size: 268435456 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715226182858529, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715226182859236, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79187, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77450, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71759, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1715226182, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "97f49480-b0d1-458b-be71-a7327b93913d", "db_session_id": "NE8VUQ9S4MFHNQMH420G"}} 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715226182859652, "job": 1, "event": "recovery_finished"} 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x56060f5a4a80 2024-05-09T03:43:03.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: DB pointer 0x56060f5b4000 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: ** DB Stats ** 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:43:03.025 INFO:journalctl@ceph.mon.a.smithi078.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: Cumulative compaction: 0.00 GB write, 24.93 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: Interval compaction: 0.00 GB write, 24.93 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T03:43:03.026 INFO:journalctl@ceph.mon.a.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:43:03.027 INFO:journalctl@ceph.mon.a.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout: Cumulative compaction: 0.00 GB write, 24.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: starting mon.a rank 0 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: mon.a@-1(???) e1 preinit fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: mon.a@-1(???).mds e1 new map 2024-05-09T03:43:03.028 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: mon.a@-1(???).mds e1 print_map 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout: e1 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout: legacy client fscid: -1 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout: No filesystems configured 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: expand_channel_meta expand map: {default=false} 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: expand_channel_meta from 'false' to 'false' 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: expand_channel_meta expanded map: {default=false} 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: expand_channel_meta expand map: {default=info} 2024-05-09T03:43:03.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: expand_channel_meta from 'info' to 'info' 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: expand_channel_meta expanded map: {default=info} 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: expand_channel_meta expand map: {default=daemon} 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: expand_channel_meta from 'daemon' to 'daemon' 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: expand_channel_meta expanded map: {default=daemon} 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: expand_channel_meta expand map: {default=debug} 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: expand_channel_meta from 'debug' to 'debug' 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: expand_channel_meta expanded map: {default=debug} 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 bash[22887]: de97a71c7cb50369d585b52c272637b11e40f5dd85f89fb697ce1dcef897d9ee 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 systemd[1]: Started Ceph mon.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: monmap e1: 1 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: fsmap 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: osdmap e1: 0 total, 0 up, 0 in 2024-05-09T03:43:03.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:02 smithi078 ceph-mon[22905]: mgrmap e1: no daemons active 2024-05-09T03:43:03.907 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote config to /etc/ceph/ceph.conf 2024-05-09T03:43:03.907 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-05-09T03:43:03.907 INFO:teuthology.orchestra.run.smithi078.stderr:Creating mgr... 2024-05-09T03:43:03.907 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying port 9283 ... 2024-05-09T03:43:04.075 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to reset failed state of unit ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.y.service: Unit ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.y.service not loaded. 2024-05-09T03:43:04.082 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph-2777c79e-0db6-11ef-bc97-c7b262605968.target.wants/ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-2777c79e-0db6-11ef-bc97-c7b262605968@.service. 2024-05-09T03:43:04.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:04 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2340621346' entity='client.admin' 2024-05-09T03:43:04.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:04 smithi078 podman[23124]: 2024-05-09 03:43:04.462905167 +0000 UTC m=+0.119088484 container create b2a9d11f22f2540da5bf9578f4559da1ec730c6e98fad3fec6a7a20846afe592 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., version=8, name=centos-stream, vcs-type=git, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-09T03:43:04.719 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-05-09T03:43:04.727 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-05-09T03:43:04.727 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-05-09T03:43:04.727 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to enable service . firewalld.service is not available 2024-05-09T03:43:04.735 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-05-09T03:43:04.743 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-05-09T03:43:04.743 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-05-09T03:43:04.743 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-05-09T03:43:04.744 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr to start... 2024-05-09T03:43:04.744 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr... 2024-05-09T03:43:04.802 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:04 smithi078 podman[23124]: 2024-05-09 03:43:04.64837758 +0000 UTC m=+0.304560896 container init b2a9d11f22f2540da5bf9578f4559da1ec730c6e98fad3fec6a7a20846afe592 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y, version=8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, ceph=True, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, release=754, architecture=x86_64, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-09T03:43:04.803 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:04 smithi078 podman[23124]: 2024-05-09 03:43:04.651140376 +0000 UTC m=+0.307323693 container start b2a9d11f22f2540da5bf9578f4559da1ec730c6e98fad3fec6a7a20846afe592 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, distribution-scope=public, ceph=True, name=centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.openshift.expose-services=, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8) 2024-05-09T03:43:04.803 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:04 smithi078 bash[23124]: b2a9d11f22f2540da5bf9578f4559da1ec730c6e98fad3fec6a7a20846afe592 2024-05-09T03:43:04.803 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:04 smithi078 systemd[1]: Started Ceph mgr.y for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:43:05.094 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:04 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:04.801+0000 7f29fc8df000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-09T03:43:05.094 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:04 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:04.932+0000 7f29fc8df000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:43:05.094 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:05.034+0000 7f29fc8df000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-09T03:43:05.272 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-05-09T03:43:05.272 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-05-09T03:43:05.272 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "2777c79e-0db6-11ef-bc97-c7b262605968", 2024-05-09T03:43:05.272 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-05-09T03:43:05.272 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-09T03:43:05.272 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-05-09T03:43:05.272 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-05-09T03:43:05.272 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:05.272 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "a" 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-09T03:43:05.273 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-09T03:43:05.274 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-09T03:43:05.274 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-09T03:43:05.274 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-09T03:43:05.277 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-09T03:43:05.277 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-09T03:43:05.277 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:05.277 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-05-09T03:43:05.277 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-09T03:43:05.277 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-09T03:43:05.278 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-05-09T03:42:58.515247+0000", 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-09T03:43:05.279 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-05-09T03:43:05.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:05 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/4259528713' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-09T03:43:05.762 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:05.353+0000 7f29fc8df000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-09T03:43:05.762 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:05.437+0000 7f29fc8df000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-09T03:43:05.790 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (1/15)... 2024-05-09T03:43:07.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:06 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:06.577+0000 7f29fc8df000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-09T03:43:07.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:06 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:06.741+0000 7f29fc8df000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-09T03:43:07.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:06 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:06.823+0000 7f29fc8df000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-09T03:43:07.335 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:07.149+0000 7f29fc8df000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-09T03:43:07.335 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:07.239+0000 7f29fc8df000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-09T03:43:07.336 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:07.334+0000 7f29fc8df000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-09T03:43:07.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:07.584+0000 7f29fc8df000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:43:08.200 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:08 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:08.044+0000 7f29fc8df000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-09T03:43:08.335 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "2777c79e-0db6-11ef-bc97-c7b262605968", 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-09T03:43:08.336 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "a" 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-09T03:43:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-05-09T03:43:08.338 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:08.339 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-05-09T03:43:08.340 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-09T03:43:08.340 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-05-09T03:42:58.515247+0000", 2024-05-09T03:43:08.340 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-05-09T03:43:08.340 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:08.340 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-09T03:43:08.340 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-05-09T03:43:08.865 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:08 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:08.543+0000 7f29fc8df000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-09T03:43:08.865 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:08 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:08.639+0000 7f29fc8df000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-09T03:43:09.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:08 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/529671142' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-09T03:43:09.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:08 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:08.932+0000 7f29fc8df000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-09T03:43:09.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:09.020+0000 7f29fc8df000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-09T03:43:09.758 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:09.418+0000 7f29fc8df000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-09T03:43:09.758 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:09.551+0000 7f29fc8df000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-09T03:43:09.758 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:09.643+0000 7f29fc8df000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-09T03:43:09.758 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:09.756+0000 7f29fc8df000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-09T03:43:10.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:09.846+0000 7f29fc8df000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-09T03:43:10.756 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:10.391+0000 7f29fc8df000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-09T03:43:11.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:10.753+0000 7f29fc8df000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-09T03:43:11.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: Activating manager daemon y 2024-05-09T03:43:11.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: mgrmap e2: y(active, starting, since 0.00307125s) 2024-05-09T03:43:11.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: from='mgr.14100 172.21.15.78:0/199089654' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T03:43:11.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: from='mgr.14100 172.21.15.78:0/199089654' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T03:43:11.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: from='mgr.14100 172.21.15.78:0/199089654' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T03:43:11.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: from='mgr.14100 172.21.15.78:0/199089654' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:43:11.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: from='mgr.14100 172.21.15.78:0/199089654' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T03:43:11.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: Manager daemon y is now available 2024-05-09T03:43:11.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: from='mgr.14100 172.21.15.78:0/199089654' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:43:11.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: from='mgr.14100 172.21.15.78:0/199089654' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:43:11.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: from='mgr.14100 172.21.15.78:0/199089654' entity='mgr.y' 2024-05-09T03:43:11.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: from='mgr.14100 172.21.15.78:0/199089654' entity='mgr.y' 2024-05-09T03:43:11.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:10 smithi078 ceph-mon[22905]: from='mgr.14100 172.21.15.78:0/199089654' entity='mgr.y' 2024-05-09T03:43:12.326 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (2/15)... 2024-05-09T03:43:13.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:12 smithi078 ceph-mon[22905]: mgrmap e3: y(active, since 1.00673s) 2024-05-09T03:43:14.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:13 smithi078 ceph-mon[22905]: mgrmap e4: y(active, since 2s) 2024-05-09T03:43:18.859 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-05-09T03:43:18.859 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-05-09T03:43:18.859 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "2777c79e-0db6-11ef-bc97-c7b262605968", 2024-05-09T03:43:18.859 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-05-09T03:43:18.859 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-09T03:43:18.859 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-05-09T03:43:18.859 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-05-09T03:43:18.859 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "a" 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 15, 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-05-09T03:43:18.860 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-09T03:43:18.861 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-09T03:43:18.861 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-09T03:43:18.861 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-09T03:43:18.861 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-09T03:43:18.864 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-09T03:43:18.864 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-09T03:43:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-05-09T03:43:18.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-09T03:43:18.867 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-05-09T03:42:58.515247+0000", 2024-05-09T03:43:18.867 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-05-09T03:43:18.867 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-05-09T03:43:18.867 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-09T03:43:18.867 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-05-09T03:43:19.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:18 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/690284726' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-09T03:43:21.046 INFO:teuthology.orchestra.run.smithi078.stderr:mgr is available 2024-05-09T03:43:21.639 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-05-09T03:43:21.639 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [global] 2024-05-09T03:43:21.639 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: fsid = 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:43:21.639 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-09T03:43:21.640 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-09T03:43:21.640 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-09T03:43:21.640 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-09T03:43:21.640 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-05-09T03:43:21.640 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [mgr] 2024-05-09T03:43:21.640 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-09T03:43:21.640 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-05-09T03:43:21.640 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [osd] 2024-05-09T03:43:21.640 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-09T03:43:21.640 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-09T03:43:21.954 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:21 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1165262664' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-09T03:43:21.954 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:21 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1165262664' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-09T03:43:22.213 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling cephadm module... 2024-05-09T03:43:23.146 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:22 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/669481794' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-05-09T03:43:23.147 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:22 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ignoring --setuser ceph since I am not root 2024-05-09T03:43:23.147 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:22 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ignoring --setgroup ceph since I am not root 2024-05-09T03:43:23.147 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:23.016+0000 7f07af865000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-09T03:43:23.147 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:23.145+0000 7f07af865000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:43:23.467 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:23.244+0000 7f07af865000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-09T03:43:23.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:23.552+0000 7f07af865000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-09T03:43:23.762 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:23.635+0000 7f07af865000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-09T03:43:24.202 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:23 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/669481794' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-05-09T03:43:24.202 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:23 smithi078 ceph-mon[22905]: mgrmap e5: y(active, since 12s) 2024-05-09T03:43:24.203 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-05-09T03:43:24.203 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 5, 2024-05-09T03:43:24.203 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-05-09T03:43:24.203 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "active_name": "y", 2024-05-09T03:43:24.203 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-09T03:43:24.204 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-05-09T03:43:24.751 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for the mgr to restart... 2024-05-09T03:43:24.751 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr epoch 5... 2024-05-09T03:43:25.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:24 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2558950052' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-09T03:43:25.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:24 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:24.791+0000 7f07af865000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-09T03:43:25.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:24 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:24.957+0000 7f07af865000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-09T03:43:25.356 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:25 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:25.041+0000 7f07af865000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-09T03:43:25.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:25 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:25.355+0000 7f07af865000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-09T03:43:25.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:25 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:25.444+0000 7f07af865000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-09T03:43:25.762 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:25 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:25.534+0000 7f07af865000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-09T03:43:26.204 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:25 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:25.770+0000 7f07af865000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:43:26.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:26 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:26.204+0000 7f07af865000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-09T03:43:27.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:26 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:26.692+0000 7f07af865000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-09T03:43:27.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:26 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:26.784+0000 7f07af865000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-09T03:43:27.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:27 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:27.069+0000 7f07af865000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-09T03:43:27.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:27 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:27.156+0000 7f07af865000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-09T03:43:27.883 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:27 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:27.551+0000 7f07af865000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-09T03:43:27.883 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:27 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:27.681+0000 7f07af865000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-09T03:43:27.883 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:27 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:27.771+0000 7f07af865000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-09T03:43:27.883 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:27 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:27.882+0000 7f07af865000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-09T03:43:28.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:27 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:27.969+0000 7f07af865000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-09T03:43:28.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:28 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:28.506+0000 7f07af865000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-09T03:43:29.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:28 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:28.856+0000 7f07af865000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-09T03:43:29.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:28 smithi078 ceph-mon[22905]: Active manager daemon y restarted 2024-05-09T03:43:29.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:28 smithi078 ceph-mon[22905]: Activating manager daemon y 2024-05-09T03:43:29.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:28 smithi078 ceph-mon[22905]: osdmap e2: 0 total, 0 up, 0 in 2024-05-09T03:43:29.882 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-05-09T03:43:29.883 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-05-09T03:43:29.883 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "initialized": true 2024-05-09T03:43:29.883 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-05-09T03:43:30.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: mgrmap e6: y(active, starting, since 0.011721s) 2024-05-09T03:43:30.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:43:30.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T03:43:30.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T03:43:30.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T03:43:30.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T03:43:30.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: Manager daemon y is now available 2024-05-09T03:43:30.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:30.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:30.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:43:30.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:43:30.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:43:30.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:43:30.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:29 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:43:30.466 INFO:teuthology.orchestra.run.smithi078.stderr:mgr epoch 5 is available 2024-05-09T03:43:30.466 INFO:teuthology.orchestra.run.smithi078.stderr:Setting orchestrator backend to cephadm... 2024-05-09T03:43:30.708 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:30 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:43:30] ENGINE Bus STARTING 2024-05-09T03:43:30.708 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:30 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:43:30] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T03:43:30.708 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:30 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:43:30] ENGINE Bus STARTED 2024-05-09T03:43:31.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:30 smithi078 ceph-mon[22905]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-05-09T03:43:31.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:30 smithi078 ceph-mon[22905]: mgrmap e7: y(active, since 1.01634s) 2024-05-09T03:43:31.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:30 smithi078 ceph-mon[22905]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-05-09T03:43:31.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:30 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:31.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:30 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:43:31.989 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:31 smithi078 ceph-mon[22905]: [09/May/2024:03:43:30] ENGINE Bus STARTING 2024-05-09T03:43:32.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:31 smithi078 ceph-mon[22905]: [09/May/2024:03:43:30] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T03:43:32.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:31 smithi078 ceph-mon[22905]: [09/May/2024:03:43:30] ENGINE Bus STARTED 2024-05-09T03:43:32.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:31 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:32.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:31 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:43:32.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:31 smithi078 ceph-mon[22905]: mgrmap e8: y(active, since 2s) 2024-05-09T03:43:32.560 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: value unchanged 2024-05-09T03:43:32.811 INFO:teuthology.orchestra.run.smithi078.stderr:Generating ssh key... 2024-05-09T03:43:33.136 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:32 smithi078 ceph-mon[22905]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:43:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:33 smithi078 ceph-mon[22905]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:43:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:33 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:34.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:33 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:34.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: Generating public/private rsa key pair. 2024-05-09T03:43:34.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: Your identification has been saved in /tmp/tmp9ap40gdq/key. 2024-05-09T03:43:34.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: Your public key has been saved in /tmp/tmp9ap40gdq/key.pub. 2024-05-09T03:43:34.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: The key fingerprint is: 2024-05-09T03:43:34.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: SHA256:VSlJ74hp2PHpa1EqwQ8clkxCLiJRXoDVl9YMuKd/ZvY ceph-2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:43:34.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: The key's randomart image is: 2024-05-09T03:43:34.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: +---[RSA 3072]----+ 2024-05-09T03:43:34.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: |.++o.o+B.o.... | 2024-05-09T03:43:34.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: |.o .o.+.B oo. | 2024-05-09T03:43:34.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: |. o .+.+..... | 2024-05-09T03:43:34.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: | . ....o=* +. | 2024-05-09T03:43:34.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: | o. S++o. | 2024-05-09T03:43:34.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: | . ...+ | 2024-05-09T03:43:34.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: | . ... | 2024-05-09T03:43:34.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: | . = .. | 2024-05-09T03:43:34.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: | = oE | 2024-05-09T03:43:34.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: +----[SHA256]-----+ 2024-05-09T03:43:34.866 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDW6hiHxs72VTaaeBs/LgZ/51ypQbvNUix6kyDKR1h6vgMiJrStpDbbhHKVaYfSiBP2hc6gWtcsfLjtwckxbWiDZo9klOc5wMwRqdeBpQcR4bebQY3/MmrWB4FiX5Ng05nhIeFd9dH4kaOYs77u6ZgJ/WR9BTvTNU1q209zD8kR6WZ1AXcE5FMqb8xpdzxmZvlEJG6pUjnb1ZdPpluCTP93oGXGusOe3OmDnAn8tqc1ZPPsDRxImkGHov4p2ZQu1NZ7NiLnctq+iFfpMjRcBmv3/g+h23Mlo+q18CsnwU1W9vl2/TP4MGMO3W6yhqZFRb7KuMES2LkSdt9NizeWXXVIY49F3XMbgF30l1NEMyXOMOr4rry2fNjR2ceCY7MRIvY5OGkAUC+rsNQDpgCeJ/d9E4XdTefBnFQaaV3Cs5UaaV5wk26PvmlyoVZq8CVaCH6ltlpu0E6GceZ7/NvGj25VbpBu5dxFlsyIB9SjZBw1lE20WwR+CziWlxzOg9HKKCM= ceph-2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:43:35.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:34 smithi078 ceph-mon[22905]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:43:35.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:34 smithi078 ceph-mon[22905]: Generating ssh key... 2024-05-09T03:43:35.416 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-05-09T03:43:35.416 INFO:teuthology.orchestra.run.smithi078.stderr:Adding key to root@localhost authorized_keys... 2024-05-09T03:43:35.416 INFO:teuthology.orchestra.run.smithi078.stderr:Adding host smithi078... 2024-05-09T03:43:36.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:35 smithi078 ceph-mon[22905]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:43:36.869 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Added host 'smithi078' with addr '172.21.15.78' 2024-05-09T03:43:37.540 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying unmanaged mon service... 2024-05-09T03:43:38.129 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:37 smithi078 ceph-mon[22905]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi078", "addr": "172.21.15.78", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:43:38.129 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:37 smithi078 ceph-mon[22905]: Deploying cephadm binary to smithi078 2024-05-09T03:43:38.129 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:37 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:38.129 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:37 smithi078 ceph-mon[22905]: Added host smithi078 2024-05-09T03:43:38.129 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:37 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:43:38.129 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled mon update... 2024-05-09T03:43:38.840 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying unmanaged mgr service... 2024-05-09T03:43:39.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:39 smithi078 ceph-mon[22905]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:43:39.215 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:39 smithi078 ceph-mon[22905]: Saving service mon spec with placement count:5 2024-05-09T03:43:39.215 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:39 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:40.426 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-05-09T03:43:40.997 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:40 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:40.997 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:40 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:40.997 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:40 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:41.814 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:41 smithi078 ceph-mon[22905]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:43:41.814 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:41 smithi078 ceph-mon[22905]: Saving service mgr spec with placement count:2 2024-05-09T03:43:41.815 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:41 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:41.815 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:41 smithi078 ceph-mon[22905]: from='mgr.14118 172.21.15.78:0/2755503250' entity='mgr.y' 2024-05-09T03:43:43.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:43 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3098627743' entity='client.admin' 2024-05-09T03:43:44.763 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling the dashboard module... 2024-05-09T03:43:45.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:45 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1913761554' entity='client.admin' 2024-05-09T03:43:46.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:46 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2705669475' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-05-09T03:43:46.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:46 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ignoring --setuser ceph since I am not root 2024-05-09T03:43:46.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:46 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ignoring --setgroup ceph since I am not root 2024-05-09T03:43:46.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:46 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:46.413+0000 7f2546858000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-09T03:43:46.852 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:46 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:46.542+0000 7f2546858000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:43:46.852 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:46 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:46.641+0000 7f2546858000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-09T03:43:47.138 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:46 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:46.957+0000 7f2546858000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-09T03:43:47.138 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:47 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:47.041+0000 7f2546858000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-09T03:43:47.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:47 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2705669475' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-05-09T03:43:47.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:47 smithi078 ceph-mon[22905]: mgrmap e9: y(active, since 17s) 2024-05-09T03:43:47.603 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-05-09T03:43:47.603 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 9, 2024-05-09T03:43:47.603 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-05-09T03:43:47.604 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "active_name": "y", 2024-05-09T03:43:47.604 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-09T03:43:47.604 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-05-09T03:43:48.148 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for the mgr to restart... 2024-05-09T03:43:48.148 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr epoch 9... 2024-05-09T03:43:48.406 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:48 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2560147493' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-09T03:43:48.406 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:48 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:48.195+0000 7f2546858000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-09T03:43:48.406 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:48 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:48.360+0000 7f2546858000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-09T03:43:48.762 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:48 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:48.446+0000 7f2546858000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-09T03:43:49.205 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:48 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:48.763+0000 7f2546858000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-09T03:43:49.205 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:48 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:48.858+0000 7f2546858000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-09T03:43:49.205 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:48 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:48.951+0000 7f2546858000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-09T03:43:49.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:49.204+0000 7f2546858000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:43:50.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:49.650+0000 7f2546858000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-09T03:43:50.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:50 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:50.138+0000 7f2546858000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-09T03:43:50.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:50 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:50.233+0000 7f2546858000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-09T03:43:51.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:50 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:50.529+0000 7f2546858000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-09T03:43:51.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:50 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:50.619+0000 7f2546858000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-09T03:43:51.367 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:51 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:51.028+0000 7f2546858000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-09T03:43:51.367 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:51 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:51.161+0000 7f2546858000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-09T03:43:51.368 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:51 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:51.252+0000 7f2546858000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-09T03:43:51.368 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:51 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:51.366+0000 7f2546858000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-09T03:43:51.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:51 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:51.455+0000 7f2546858000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-09T03:43:52.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:52.000+0000 7f2546858000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-09T03:43:52.610 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:43:52.358+0000 7f2546858000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-09T03:43:53.364 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:43:53] ENGINE Bus STARTING 2024-05-09T03:43:53.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-05-09T03:43:53.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-05-09T03:43:53.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "initialized": true 2024-05-09T03:43:53.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-05-09T03:43:53.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:43:53] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T03:43:53.762 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:43:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:43:53] ENGINE Bus STARTED 2024-05-09T03:43:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: Active manager daemon y restarted 2024-05-09T03:43:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: Activating manager daemon y 2024-05-09T03:43:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: osdmap e3: 0 total, 0 up, 0 in 2024-05-09T03:43:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: mgrmap e10: y(active, starting, since 0.0551507s) 2024-05-09T03:43:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:43:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T03:43:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T03:43:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T03:43:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T03:43:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: Manager daemon y is now available 2024-05-09T03:43:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:43:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:43:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:43:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:43:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:53 smithi078 ceph-mon[22905]: [09/May/2024:03:43:53] ENGINE Bus STARTING 2024-05-09T03:43:54.074 INFO:teuthology.orchestra.run.smithi078.stderr:mgr epoch 9 is available 2024-05-09T03:43:54.074 INFO:teuthology.orchestra.run.smithi078.stderr:Generating a dashboard self-signed certificate... 2024-05-09T03:43:54.692 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:54 smithi078 ceph-mon[22905]: [09/May/2024:03:43:53] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T03:43:54.692 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:54 smithi078 ceph-mon[22905]: [09/May/2024:03:43:53] ENGINE Bus STARTED 2024-05-09T03:43:54.692 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:54 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:43:54.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:54 smithi078 ceph-mon[22905]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-05-09T03:43:54.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:54 smithi078 ceph-mon[22905]: mgrmap e11: y(active, since 1.05705s) 2024-05-09T03:43:54.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:54 smithi078 ceph-mon[22905]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-05-09T03:43:54.930 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Self-signed certificate created 2024-05-09T03:43:55.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:55 smithi078 ceph-mon[22905]: mgrmap e12: y(active, since 2s) 2024-05-09T03:43:55.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:55 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:43:55.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:55 smithi078 ceph-mon[22905]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:43:55.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:55 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:43:55.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:55 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:43:55.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:55 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:43:55.551 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial admin user... 2024-05-09T03:43:56.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$NZtyHj9C9b7QD0c8Etu5s.iRE2GLIKB5814ZgBn//Nar0PLypsSSa", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1715226236, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-05-09T03:43:56.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:56 smithi078 ceph-mon[22905]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:43:56.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:56 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi078.stderr:Fetching dashboard port number... 2024-05-09T03:43:58.254 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 8443 2024-05-09T03:43:58.863 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-05-09T03:43:58.870 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-05-09T03:43:58.871 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-05-09T03:43:58.871 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-05-09T03:43:58.873 INFO:teuthology.orchestra.run.smithi078.stderr:Ceph Dashboard is now available at: 2024-05-09T03:43:58.874 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-05-09T03:43:58.874 INFO:teuthology.orchestra.run.smithi078.stderr: URL: https://smithi078.front.sepia.ceph.com:8443/ 2024-05-09T03:43:58.874 INFO:teuthology.orchestra.run.smithi078.stderr: User: admin 2024-05-09T03:43:58.874 INFO:teuthology.orchestra.run.smithi078.stderr: Password: erezn62jq6 2024-05-09T03:43:58.874 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-05-09T03:43:58.874 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling autotune for osd_memory_target 2024-05-09T03:43:59.143 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:58 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1646295712' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-05-09T03:44:00.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:43:59 smithi078 ceph-mon[22905]: mgrmap e13: y(active, since 6s) 2024-05-09T03:44:00.853 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-05-09T03:44:01.511 INFO:teuthology.orchestra.run.smithi078.stderr:You can access the Ceph CLI with: 2024-05-09T03:44:01.511 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-05-09T03:44:01.511 INFO:teuthology.orchestra.run.smithi078.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-05-09T03:44:01.512 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-05-09T03:44:01.512 INFO:teuthology.orchestra.run.smithi078.stderr:Please consider enabling telemetry to help improve Ceph: 2024-05-09T03:44:01.512 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-05-09T03:44:01.512 INFO:teuthology.orchestra.run.smithi078.stderr: ceph telemetry on 2024-05-09T03:44:01.512 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-05-09T03:44:01.512 INFO:teuthology.orchestra.run.smithi078.stderr:For more information see: 2024-05-09T03:44:01.512 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-05-09T03:44:01.512 INFO:teuthology.orchestra.run.smithi078.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-05-09T03:44:01.512 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-05-09T03:44:01.512 INFO:teuthology.orchestra.run.smithi078.stderr:Bootstrap complete. 2024-05-09T03:44:01.544 INFO:tasks.cephadm:Fetching config... 2024-05-09T03:44:01.544 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-09T03:44:01.544 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-05-09T03:44:01.562 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-05-09T03:44:01.562 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-09T03:44:01.562 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-05-09T03:44:01.621 INFO:tasks.cephadm:Fetching mon keyring... 2024-05-09T03:44:01.621 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-09T03:44:01.621 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/keyring of=/dev/stdout 2024-05-09T03:44:01.686 INFO:tasks.cephadm:Fetching pub ssh key... 2024-05-09T03:44:01.686 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-09T03:44:01.686 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-05-09T03:44:01.742 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-05-09T03:44:01.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDW6hiHxs72VTaaeBs/LgZ/51ypQbvNUix6kyDKR1h6vgMiJrStpDbbhHKVaYfSiBP2hc6gWtcsfLjtwckxbWiDZo9klOc5wMwRqdeBpQcR4bebQY3/MmrWB4FiX5Ng05nhIeFd9dH4kaOYs77u6ZgJ/WR9BTvTNU1q209zD8kR6WZ1AXcE5FMqb8xpdzxmZvlEJG6pUjnb1ZdPpluCTP93oGXGusOe3OmDnAn8tqc1ZPPsDRxImkGHov4p2ZQu1NZ7NiLnctq+iFfpMjRcBmv3/g+h23Mlo+q18CsnwU1W9vl2/TP4MGMO3W6yhqZFRb7KuMES2LkSdt9NizeWXXVIY49F3XMbgF30l1NEMyXOMOr4rry2fNjR2ceCY7MRIvY5OGkAUC+rsNQDpgCeJ/d9E4XdTefBnFQaaV3Cs5UaaV5wk26PvmlyoVZq8CVaCH6ltlpu0E6GceZ7/NvGj25VbpBu5dxFlsyIB9SjZBw1lE20WwR+CziWlxzOg9HKKCM= ceph-2777c79e-0db6-11ef-bc97-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-09T03:44:01.828 INFO:teuthology.orchestra.run.smithi078.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDW6hiHxs72VTaaeBs/LgZ/51ypQbvNUix6kyDKR1h6vgMiJrStpDbbhHKVaYfSiBP2hc6gWtcsfLjtwckxbWiDZo9klOc5wMwRqdeBpQcR4bebQY3/MmrWB4FiX5Ng05nhIeFd9dH4kaOYs77u6ZgJ/WR9BTvTNU1q209zD8kR6WZ1AXcE5FMqb8xpdzxmZvlEJG6pUjnb1ZdPpluCTP93oGXGusOe3OmDnAn8tqc1ZPPsDRxImkGHov4p2ZQu1NZ7NiLnctq+iFfpMjRcBmv3/g+h23Mlo+q18CsnwU1W9vl2/TP4MGMO3W6yhqZFRb7KuMES2LkSdt9NizeWXXVIY49F3XMbgF30l1NEMyXOMOr4rry2fNjR2ceCY7MRIvY5OGkAUC+rsNQDpgCeJ/d9E4XdTefBnFQaaV3Cs5UaaV5wk26PvmlyoVZq8CVaCH6ltlpu0E6GceZ7/NvGj25VbpBu5dxFlsyIB9SjZBw1lE20WwR+CziWlxzOg9HKKCM= ceph-2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:44:01.840 DEBUG:teuthology.orchestra.run.smithi132:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDW6hiHxs72VTaaeBs/LgZ/51ypQbvNUix6kyDKR1h6vgMiJrStpDbbhHKVaYfSiBP2hc6gWtcsfLjtwckxbWiDZo9klOc5wMwRqdeBpQcR4bebQY3/MmrWB4FiX5Ng05nhIeFd9dH4kaOYs77u6ZgJ/WR9BTvTNU1q209zD8kR6WZ1AXcE5FMqb8xpdzxmZvlEJG6pUjnb1ZdPpluCTP93oGXGusOe3OmDnAn8tqc1ZPPsDRxImkGHov4p2ZQu1NZ7NiLnctq+iFfpMjRcBmv3/g+h23Mlo+q18CsnwU1W9vl2/TP4MGMO3W6yhqZFRb7KuMES2LkSdt9NizeWXXVIY49F3XMbgF30l1NEMyXOMOr4rry2fNjR2ceCY7MRIvY5OGkAUC+rsNQDpgCeJ/d9E4XdTefBnFQaaV3Cs5UaaV5wk26PvmlyoVZq8CVaCH6ltlpu0E6GceZ7/NvGj25VbpBu5dxFlsyIB9SjZBw1lE20WwR+CziWlxzOg9HKKCM= ceph-2777c79e-0db6-11ef-bc97-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-09T03:44:01.883 INFO:teuthology.orchestra.run.smithi132.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDW6hiHxs72VTaaeBs/LgZ/51ypQbvNUix6kyDKR1h6vgMiJrStpDbbhHKVaYfSiBP2hc6gWtcsfLjtwckxbWiDZo9klOc5wMwRqdeBpQcR4bebQY3/MmrWB4FiX5Ng05nhIeFd9dH4kaOYs77u6ZgJ/WR9BTvTNU1q209zD8kR6WZ1AXcE5FMqb8xpdzxmZvlEJG6pUjnb1ZdPpluCTP93oGXGusOe3OmDnAn8tqc1ZPPsDRxImkGHov4p2ZQu1NZ7NiLnctq+iFfpMjRcBmv3/g+h23Mlo+q18CsnwU1W9vl2/TP4MGMO3W6yhqZFRb7KuMES2LkSdt9NizeWXXVIY49F3XMbgF30l1NEMyXOMOr4rry2fNjR2ceCY7MRIvY5OGkAUC+rsNQDpgCeJ/d9E4XdTefBnFQaaV3Cs5UaaV5wk26PvmlyoVZq8CVaCH6ltlpu0E6GceZ7/NvGj25VbpBu5dxFlsyIB9SjZBw1lE20WwR+CziWlxzOg9HKKCM= ceph-2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:44:01.895 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-05-09T03:44:01.917 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:01 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/774142859' entity='client.admin' 2024-05-09T03:44:04.442 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:04 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:04.442 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:04 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:44:04.442 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:04 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:04.442 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:04 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:04.442 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:04 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:05.746 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-05-09T03:44:05.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-05-09T03:44:06.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:05 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/780926721' entity='client.admin' 2024-05-09T03:44:06.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:06.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:08.420 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi132 2024-05-09T03:44:08.420 DEBUG:teuthology.orchestra.run.smithi132:> set -ex 2024-05-09T03:44:08.420 DEBUG:teuthology.orchestra.run.smithi132:> dd of=/etc/ceph/ceph.conf 2024-05-09T03:44:08.439 DEBUG:teuthology.orchestra.run.smithi132:> set -ex 2024-05-09T03:44:08.440 DEBUG:teuthology.orchestra.run.smithi132:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:44:08.497 INFO:tasks.cephadm:Adding host smithi132 to orchestrator... 2024-05-09T03:44:08.497 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch host add smithi132 2024-05-09T03:44:08.952 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:08 smithi078 ceph-mon[22905]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:44:08.952 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:08.952 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:08.952 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:08.953 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:08.953 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:08 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T03:44:08.953 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:08 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:44:08.953 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:08.953 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:10.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:10 smithi078 ceph-mon[22905]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi132", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:44:11.136 INFO:teuthology.orchestra.run.smithi078.stdout:Added host 'smithi132' with addr '172.21.15.132' 2024-05-09T03:44:11.706 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch host ls --format=json 2024-05-09T03:44:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:12 smithi078 ceph-mon[22905]: Deploying cephadm binary to smithi132 2024-05-09T03:44:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:12 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:12 smithi078 ceph-mon[22905]: Added host smithi132 2024-05-09T03:44:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:12 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:12 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:12 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:12 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:12 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:13.504 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:44:13.504 INFO:teuthology.orchestra.run.smithi078.stdout:[{"addr": "172.21.15.78", "hostname": "smithi078", "labels": [], "status": ""}, {"addr": "172.21.15.132", "hostname": "smithi132", "labels": [], "status": ""}] 2024-05-09T03:44:14.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:13 smithi078 ceph-mon[22905]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:14.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:13 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:14.103 INFO:tasks.cephadm:Setting crush tunables to default 2024-05-09T03:44:14.103 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd crush tunables default 2024-05-09T03:44:15.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:14 smithi078 ceph-mon[22905]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-09T03:44:16.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:15 smithi078 ceph-mon[22905]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:16.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:15 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:16.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:15 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:16.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:15 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi132", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:44:16.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:15 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:16.720 INFO:teuthology.orchestra.run.smithi078.stderr:adjusted tunables profile to default 2024-05-09T03:44:16.998 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:16 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3746839152' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-05-09T03:44:17.293 INFO:tasks.cephadm:Adding mon.a on smithi078 2024-05-09T03:44:17.293 INFO:tasks.cephadm:Adding mon.c on smithi078 2024-05-09T03:44:17.293 INFO:tasks.cephadm:Adding mon.b on smithi132 2024-05-09T03:44:17.294 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch apply mon '3;smithi078:172.21.15.78=a;smithi078:[v2:172.21.15.78:3301,v1:172.21.15.78:6790]=c;smithi132:172.21.15.132=b' 2024-05-09T03:44:18.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:17 smithi078 ceph-mon[22905]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:18.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:17 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3746839152' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-05-09T03:44:18.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:17 smithi078 ceph-mon[22905]: osdmap e4: 0 total, 0 up, 0 in 2024-05-09T03:44:19.056 INFO:teuthology.orchestra.run.smithi132.stdout:Scheduled mon update... 2024-05-09T03:44:19.643 DEBUG:teuthology.orchestra.run.smithi078:mon.c> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.c.service 2024-05-09T03:44:19.646 DEBUG:teuthology.orchestra.run.smithi132:mon.b> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.b.service 2024-05-09T03:44:19.648 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-09T03:44:19.648 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-09T03:44:20.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:20.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi078:172.21.15.78=a;smithi078:[v2:172.21.15.78:3301,v1:172.21.15.78:6790]=c;smithi132:172.21.15.132=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:44:20.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: Saving service mon spec with placement smithi078:172.21.15.78=a;smithi078:[v2:172.21.15.78:3301,v1:172.21.15.78:6790]=c;smithi132:172.21.15.132=b;count:3 2024-05-09T03:44:20.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:20.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:20.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:20.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:20.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T03:44:20.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:44:20.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:20.136 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:20.136 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:20.136 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:20 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:21 smithi078 ceph-mon[22905]: Deploying daemon mon.c on smithi078 2024-05-09T03:44:21.390 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:44:21.390 INFO:teuthology.orchestra.run.smithi132.stdout:{"epoch":1,"fsid":"2777c79e-0db6-11ef-bc97-c7b262605968","modified":"2024-05-09T03:42:55.620159Z","created":"2024-05-09T03:42:55.620159Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-09T03:44:21.392 INFO:teuthology.orchestra.run.smithi132.stderr:dumped monmap epoch 1 2024-05-09T03:44:22.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[22905]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:22.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[22905]: from='client.? 172.21.15.132:0/194346120' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-09T03:44:22.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 podman[27611]: 2024-05-09 03:44:22.128750045 +0000 UTC m=+0.018886983 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-09T03:44:22.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 podman[27611]: 2024-05-09 03:44:22.431846002 +0000 UTC m=+0.321982929 container init 7563b8437b5529f1d4f1bb293790f9c3c375a5da74f7588b36c3e831d10964b9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, com.redhat.component=centos-stream-container) 2024-05-09T03:44:22.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 podman[27611]: 2024-05-09 03:44:22.435147413 +0000 UTC m=+0.325284333 container start 7563b8437b5529f1d4f1bb293790f9c3c375a5da74f7588b36c3e831d10964b9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , name=centos-stream, ceph=True, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, RELEASE=HEAD, version=8, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.buildah.version=1.19.8) 2024-05-09T03:44:22.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: set uid:gid to 167:167 (ceph:ceph) 2024-05-09T03:44:22.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: pidfile_write: ignore empty --pid-file 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: load: jerasure load: lrc 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: RocksDB version: 6.15.5 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Compile date Apr 18 2022 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: DB SUMMARY 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: DB Session ID: Q2Z2YE620NSCFGI22ASR 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: CURRENT file: CURRENT 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: IDENTITY file: IDENTITY 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.error_if_exists: 0 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.create_if_missing: 0 2024-05-09T03:44:22.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.paranoid_checks: 1 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.env: 0x55655b8c6860 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.fs: Posix File System 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.info_log: 0x55655c821dc0 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_file_opening_threads: 16 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.statistics: (nil) 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.use_fsync: 0 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_log_file_size: 0 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.keep_log_file_num: 1000 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.recycle_log_file_num: 0 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.allow_fallocate: 1 2024-05-09T03:44:22.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.allow_mmap_reads: 0 2024-05-09T03:44:22.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.allow_mmap_writes: 0 2024-05-09T03:44:22.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.use_direct_reads: 0 2024-05-09T03:44:22.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-09T03:44:22.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.create_missing_column_families: 0 2024-05-09T03:44:22.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.db_log_dir: 2024-05-09T03:44:22.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-05-09T03:44:22.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-09T03:44:22.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-09T03:44:22.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-09T03:44:22.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-09T03:44:22.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-09T03:44:22.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.advise_random_on_open: 1 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.db_write_buffer_size: 0 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.write_buffer_manager: 0x55655c9122d0 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.rate_limiter: (nil) 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.wal_recovery_mode: 2 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.enable_thread_tracking: 0 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.enable_pipelined_write: 0 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.unordered_write: 0 2024-05-09T03:44:22.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.row_cache: None 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.wal_filter: None 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.allow_ingest_behind: 0 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.preserve_deletes: 0 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.two_write_queues: 0 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.manual_wal_flush: 0 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.atomic_flush: 0 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-09T03:44:22.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.log_readahead_size: 0 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.best_efforts_recovery: 0 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.allow_data_in_errors: 0 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.db_host_id: __hostname__ 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_background_jobs: 2 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_background_compactions: -1 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_subcompactions: 1 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_total_wal_size: 0 2024-05-09T03:44:22.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_open_files: -1 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.bytes_per_sync: 0 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_readahead_size: 0 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_background_flushes: -1 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Compression algorithms supported: 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: kZSTD supported: 0 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: kXpressCompression supported: 0 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: kLZ4HCCompression supported: 1 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: kLZ4Compression supported: 1 2024-05-09T03:44:22.580 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: kBZip2Compression supported: 0 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: kZlibCompression supported: 1 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: kSnappyCompression supported: 1 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.merge_operator: 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_filter: None 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_filter_factory: None 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.sst_partitioner_factory: None 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55655c7edd00) 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-09T03:44:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: index_type: 0 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: data_block_index_type: 0 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: index_shortening: 1 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: hash_index_allow_collision: 1 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: checksum: 1 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: no_block_cache: 0 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_cache: 0x55655c858170 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_cache_options: 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: capacity : 536870912 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: num_shard_bits : 4 2024-05-09T03:44:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout: strict_capacity_limit : 0 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_cache_compressed: (nil) 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: persistent_cache: (nil) 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_size: 4096 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_size_deviation: 10 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_restart_interval: 16 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: index_block_restart_interval: 1 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: metadata_block_size: 4096 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: partition_filters: 0 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: use_delta_encoding: 1 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: whole_key_filtering: 1 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: verify_compression: 0 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: format_version: 4 2024-05-09T03:44:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout: enable_index_compression: 1 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_align: 0 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.write_buffer_size: 33554432 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_write_buffer_number: 2 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compression: NoCompression 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.bottommost_compression: Disabled 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.prefix_extractor: nullptr 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.num_levels: 7 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-09T03:44:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compression_opts.level: 32767 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compression_opts.strategy: 0 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compression_opts.enabled: false 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.target_file_size_base: 67108864 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-09T03:44:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.arena_block_size: 4194304 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-09T03:44:22.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.disable_auto_compactions: 0 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.table_properties_collectors: 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.inplace_update_support: 0 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-09T03:44:22.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.bloom_locality: 0 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.max_successive_merges: 0 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.paranoid_file_checks: 0 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.force_consistency_checks: 1 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.report_bg_io_stats: 0 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.ttl: 2592000 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.enable_blob_files: false 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.min_blob_size: 0 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.blob_file_size: 268435456 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-09T03:44:22.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715226262461803, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715226262462111, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1715226262, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "bec0f1ab-61ff-437e-a555-42fb2e7af8a0", "db_session_id": "Q2Z2YE620NSCFGI22ASR"}} 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715226262462417, "job": 1, "event": "recovery_finished"} 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55655c83ea80 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: DB pointer 0x55655c84e000 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-09T03:44:22.589 INFO:journalctl@ceph.mon.c.smithi078.stdout: ** DB Stats ** 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:22.590 INFO:journalctl@ceph.mon.c.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: Interval compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-09T03:44:22.591 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-09T03:44:22.592 INFO:journalctl@ceph.mon.c.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: using public_addrv [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: starting mon.c rank -1 at public addrs [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] at bind addrs [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(???) e0 preinit fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).mds e1 new map 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).mds e1 print_map 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout: e1 2024-05-09T03:44:22.593 INFO:journalctl@ceph.mon.c.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout: legacy client fscid: -1 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout: No filesystems configured 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-09T03:44:22.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi078:172.21.15.78=a;smithi078:[v2:172.21.15.78:3301,v1:172.21.15.78:6790]=c;smithi132:172.21.15.132=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: Saving service mon spec with placement smithi078:172.21.15.78=a;smithi078:[v2:172.21.15.78:3301,v1:172.21.15.78:6790]=c;smithi132:172.21.15.132=b;count:3 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: Deploying daemon mon.c on smithi078 2024-05-09T03:44:22.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:22.596 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: from='client.? 172.21.15.132:0/194346120' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-09T03:44:22.596 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-05-09T03:44:22.596 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 ceph-mon[27629]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-05-09T03:44:22.596 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 bash[27611]: 7563b8437b5529f1d4f1bb293790f9c3c375a5da74f7588b36c3e831d10964b9 2024-05-09T03:44:22.596 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:22 smithi078 systemd[1]: Started Ceph mon.c for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:44:22.983 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-09T03:44:22.984 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-09T03:44:27.795 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:44:27.795 INFO:teuthology.orchestra.run.smithi132.stdout:{"epoch":2,"fsid":"2777c79e-0db6-11ef-bc97-c7b262605968","modified":"2024-05-09T03:44:22.479963Z","created":"2024-05-09T03:42:55.620159Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3301","nonce":0},{"type":"v1","addr":"172.21.15.78:6790","nonce":0}]},"addr":"172.21.15.78:6790/0","public_addr":"172.21.15.78:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-09T03:44:27.796 INFO:teuthology.orchestra.run.smithi132.stderr:dumped monmap epoch 2 2024-05-09T03:44:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:44:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: mon.a calling monitor election 2024-05-09T03:44:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: mon.c calling monitor election 2024-05-09T03:44:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: monmap e2: 2 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: fsmap 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: osdmap e4: 0 total, 0 up, 0 in 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: mgrmap e13: y(active, since 35s) 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: overall HEALTH_OK 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: mon.a calling monitor election 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: mon.c calling monitor election 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: monmap e2: 2 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} 2024-05-09T03:44:28.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: fsmap 2024-05-09T03:44:28.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: osdmap e4: 0 total, 0 up, 0 in 2024-05-09T03:44:28.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: mgrmap e13: y(active, since 35s) 2024-05-09T03:44:28.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: overall HEALTH_OK 2024-05-09T03:44:28.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:28.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:28.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:27 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:29.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:28 smithi078 ceph-mon[22905]: Deploying daemon mon.b on smithi132 2024-05-09T03:44:29.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:28 smithi078 ceph-mon[22905]: from='client.? 172.21.15.132:0/2069902379' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-09T03:44:29.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:28 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:29.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:28 smithi078 ceph-mon[27629]: Deploying daemon mon.b on smithi132 2024-05-09T03:44:29.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:28 smithi078 ceph-mon[27629]: from='client.? 172.21.15.132:0/2069902379' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-09T03:44:29.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:28 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:30.008 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-09T03:44:30.008 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-09T03:44:30.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:29 smithi078 ceph-mon[22905]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:30.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:29 smithi078 ceph-mon[27629]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:32.195 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 podman[23753]: 2024-05-09 03:44:32.100959062 +0000 UTC m=+0.304196864 container create a71bae52559df4ee8bff01c3c97e57148a5927605408fbf72ccd94fbceb27387 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , RELEASE=HEAD, io.buildah.version=1.19.8, io.openshift.expose-services=, com.redhat.component=centos-stream-container, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., ceph=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.display-name=CentOS Stream 8) 2024-05-09T03:44:32.242 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:44:32.242 INFO:teuthology.orchestra.run.smithi132.stdout:{"epoch":2,"fsid":"2777c79e-0db6-11ef-bc97-c7b262605968","modified":"2024-05-09T03:44:22.479963Z","created":"2024-05-09T03:42:55.620159Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3301","nonce":0},{"type":"v1","addr":"172.21.15.78:6790","nonce":0}]},"addr":"172.21.15.78:6790/0","public_addr":"172.21.15.78:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-09T03:44:32.243 INFO:teuthology.orchestra.run.smithi132.stderr:dumped monmap epoch 2 2024-05-09T03:44:32.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:31 smithi078 ceph-mon[22905]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:32.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:31 smithi078 ceph-mon[27629]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:32.462 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 podman[23753]: 2024-05-09 03:44:32.337593066 +0000 UTC m=+0.540830867 container init a71bae52559df4ee8bff01c3c97e57148a5927605408fbf72ccd94fbceb27387 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., ceph=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, RELEASE=HEAD, vcs-type=git, com.redhat.component=centos-stream-container, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-09T03:44:32.462 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 podman[23753]: 2024-05-09 03:44:32.341062958 +0000 UTC m=+0.544300753 container start a71bae52559df4ee8bff01c3c97e57148a5927605408fbf72ccd94fbceb27387 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, vcs-type=git, com.redhat.component=centos-stream-container, release=754, vendor=Red Hat, Inc., GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8) 2024-05-09T03:44:32.462 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: set uid:gid to 167:167 (ceph:ceph) 2024-05-09T03:44:32.462 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-09T03:44:32.462 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: pidfile_write: ignore empty --pid-file 2024-05-09T03:44:32.463 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: load: jerasure load: lrc 2024-05-09T03:44:32.463 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: RocksDB version: 6.15.5 2024-05-09T03:44:32.463 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-09T03:44:32.463 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Compile date Apr 18 2022 2024-05-09T03:44:32.463 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: DB SUMMARY 2024-05-09T03:44:32.463 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: DB Session ID: BHFT7EGGWZZ9Q2SDJ5QD 2024-05-09T03:44:32.463 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: CURRENT file: CURRENT 2024-05-09T03:44:32.463 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: IDENTITY file: IDENTITY 2024-05-09T03:44:32.463 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-05-09T03:44:32.464 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-05-09T03:44:32.464 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-05-09T03:44:32.464 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.error_if_exists: 0 2024-05-09T03:44:32.464 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.create_if_missing: 0 2024-05-09T03:44:32.464 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.paranoid_checks: 1 2024-05-09T03:44:32.464 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-09T03:44:32.464 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.env: 0x5610248c6860 2024-05-09T03:44:32.464 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.fs: Posix File System 2024-05-09T03:44:32.464 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.info_log: 0x5610251f9dc0 2024-05-09T03:44:32.465 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_file_opening_threads: 16 2024-05-09T03:44:32.465 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.statistics: (nil) 2024-05-09T03:44:32.465 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.use_fsync: 0 2024-05-09T03:44:32.465 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_log_file_size: 0 2024-05-09T03:44:32.465 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-09T03:44:32.465 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-09T03:44:32.465 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.keep_log_file_num: 1000 2024-05-09T03:44:32.465 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.recycle_log_file_num: 0 2024-05-09T03:44:32.465 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.allow_fallocate: 1 2024-05-09T03:44:32.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.allow_mmap_reads: 0 2024-05-09T03:44:32.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.allow_mmap_writes: 0 2024-05-09T03:44:32.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.use_direct_reads: 0 2024-05-09T03:44:32.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-09T03:44:32.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.create_missing_column_families: 0 2024-05-09T03:44:32.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.db_log_dir: 2024-05-09T03:44:32.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-05-09T03:44:32.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-09T03:44:32.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-09T03:44:32.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-09T03:44:32.467 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-09T03:44:32.467 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-09T03:44:32.467 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-09T03:44:32.468 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.advise_random_on_open: 1 2024-05-09T03:44:32.468 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.db_write_buffer_size: 0 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.write_buffer_manager: 0x5610252ea240 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.rate_limiter: (nil) 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.wal_recovery_mode: 2 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.enable_thread_tracking: 0 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.enable_pipelined_write: 0 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.unordered_write: 0 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-09T03:44:32.469 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.row_cache: None 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.wal_filter: None 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.allow_ingest_behind: 0 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.preserve_deletes: 0 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.two_write_queues: 0 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.manual_wal_flush: 0 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.atomic_flush: 0 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.log_readahead_size: 0 2024-05-09T03:44:32.470 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.best_efforts_recovery: 0 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.allow_data_in_errors: 0 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.db_host_id: __hostname__ 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_background_jobs: 2 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_background_compactions: -1 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_subcompactions: 1 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_total_wal_size: 0 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-09T03:44:32.471 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_open_files: -1 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.bytes_per_sync: 0 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_readahead_size: 0 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_background_flushes: -1 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Compression algorithms supported: 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: kZSTD supported: 0 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: kXpressCompression supported: 0 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: kLZ4HCCompression supported: 1 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: kLZ4Compression supported: 1 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: kBZip2Compression supported: 0 2024-05-09T03:44:32.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: kZlibCompression supported: 1 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: kSnappyCompression supported: 1 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.merge_operator: 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_filter: None 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_filter_factory: None 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.sst_partitioner_factory: None 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5610251c5d00) 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout: cache_index_and_filter_blocks: 1 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-09T03:44:32.473 INFO:journalctl@ceph.mon.b.smithi132.stdout: pin_top_level_index_and_filter: 1 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: index_type: 0 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: data_block_index_type: 0 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: index_shortening: 1 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: hash_index_allow_collision: 1 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: checksum: 1 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: no_block_cache: 0 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_cache: 0x561025230170 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_cache_name: BinnedLRUCache 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_cache_options: 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: capacity : 536870912 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: num_shard_bits : 4 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: strict_capacity_limit : 0 2024-05-09T03:44:32.474 INFO:journalctl@ceph.mon.b.smithi132.stdout: high_pri_pool_ratio: 0.000 2024-05-09T03:44:32.475 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_cache_compressed: (nil) 2024-05-09T03:44:32.475 INFO:journalctl@ceph.mon.b.smithi132.stdout: persistent_cache: (nil) 2024-05-09T03:44:32.475 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_size: 4096 2024-05-09T03:44:32.475 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_size_deviation: 10 2024-05-09T03:44:32.475 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_restart_interval: 16 2024-05-09T03:44:32.475 INFO:journalctl@ceph.mon.b.smithi132.stdout: index_block_restart_interval: 1 2024-05-09T03:44:32.475 INFO:journalctl@ceph.mon.b.smithi132.stdout: metadata_block_size: 4096 2024-05-09T03:44:32.475 INFO:journalctl@ceph.mon.b.smithi132.stdout: partition_filters: 0 2024-05-09T03:44:32.475 INFO:journalctl@ceph.mon.b.smithi132.stdout: use_delta_encoding: 1 2024-05-09T03:44:32.475 INFO:journalctl@ceph.mon.b.smithi132.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-09T03:44:32.475 INFO:journalctl@ceph.mon.b.smithi132.stdout: whole_key_filtering: 1 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout: verify_compression: 0 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout: read_amp_bytes_per_bit: 0 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout: format_version: 4 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout: enable_index_compression: 1 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_align: 0 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.write_buffer_size: 33554432 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_write_buffer_number: 2 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compression: NoCompression 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.bottommost_compression: Disabled 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.prefix_extractor: nullptr 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.num_levels: 7 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-09T03:44:32.476 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compression_opts.level: 32767 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compression_opts.strategy: 0 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compression_opts.enabled: false 2024-05-09T03:44:32.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.target_file_size_base: 67108864 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-09T03:44:32.478 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-09T03:44:32.479 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-09T03:44:32.479 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-09T03:44:32.479 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-09T03:44:32.479 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.arena_block_size: 4194304 2024-05-09T03:44:32.479 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-09T03:44:32.479 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-09T03:44:32.479 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-09T03:44:32.479 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.disable_auto_compactions: 0 2024-05-09T03:44:32.479 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-09T03:44:32.479 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-09T03:44:32.479 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-09T03:44:32.479 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.table_properties_collectors: 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.inplace_update_support: 0 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-09T03:44:32.480 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.bloom_locality: 0 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.max_successive_merges: 0 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.paranoid_file_checks: 0 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.force_consistency_checks: 1 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.report_bg_io_stats: 0 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.ttl: 2592000 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.enable_blob_files: false 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.min_blob_size: 0 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.blob_file_size: 268435456 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-09T03:44:32.481 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715226272367682, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715226272367981, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1715226272, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "75af080e-61ce-4a38-a278-0ede93e52399", "db_session_id": "BHFT7EGGWZZ9Q2SDJ5QD"}} 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715226272368283, "job": 1, "event": "recovery_finished"} 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x561025216a80 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: DB pointer 0x561025226000 2024-05-09T03:44:32.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: ** DB Stats ** 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: ** Compaction Stats [default] ** 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T03:44:32.483 INFO:journalctl@ceph.mon.b.smithi132.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: ** Compaction Stats [default] ** 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-09T03:44:32.484 INFO:journalctl@ceph.mon.b.smithi132.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: Interval compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: ** Compaction Stats [default] ** 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-09T03:44:32.485 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: ** Compaction Stats [default] ** 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T03:44:32.486 INFO:journalctl@ceph.mon.b.smithi132.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-09T03:44:32.487 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T03:44:32.487 INFO:journalctl@ceph.mon.b.smithi132.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-09T03:44:32.487 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-05-09T03:44:32.487 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: using public_addr v2:172.21.15.132:0/0 -> [v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0] 2024-05-09T03:44:32.487 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: starting mon.b rank -1 at public addrs [v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0] at bind addrs [v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:44:32.487 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(???) e0 preinit fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:44:32.487 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).mds e1 new map 2024-05-09T03:44:32.487 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).mds e1 print_map 2024-05-09T03:44:32.487 INFO:journalctl@ceph.mon.b.smithi132.stdout: e1 2024-05-09T03:44:32.487 INFO:journalctl@ceph.mon.b.smithi132.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-09T03:44:32.487 INFO:journalctl@ceph.mon.b.smithi132.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-09T03:44:32.487 INFO:journalctl@ceph.mon.b.smithi132.stdout: legacy client fscid: -1 2024-05-09T03:44:32.488 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T03:44:32.488 INFO:journalctl@ceph.mon.b.smithi132.stdout: No filesystems configured 2024-05-09T03:44:32.488 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-05-09T03:44:32.488 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-05-09T03:44:32.488 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-05-09T03:44:32.488 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-05-09T03:44:32.488 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-05-09T03:44:32.488 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-05-09T03:44:32.488 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-09T03:44:32.488 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-09T03:44:32.488 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-09T03:44:32.488 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-09T03:44:32.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:32.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi078:172.21.15.78=a;smithi078:[v2:172.21.15.78:3301,v1:172.21.15.78:6790]=c;smithi132:172.21.15.132=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:44:32.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: Saving service mon spec with placement smithi078:172.21.15.78=a;smithi078:[v2:172.21.15.78:3301,v1:172.21.15.78:6790]=c;smithi132:172.21.15.132=b;count:3 2024-05-09T03:44:32.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:32.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:32.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:32.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:32.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T03:44:32.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:44:32.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:32.490 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:32.490 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:32.490 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:32.490 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: Deploying daemon mon.c on smithi078 2024-05-09T03:44:32.490 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:32.490 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='client.? 172.21.15.132:0/194346120' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-09T03:44:32.490 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:32.490 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:44:32.490 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.a calling monitor election 2024-05-09T03:44:32.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:32.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:32.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:32.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:32.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.c calling monitor election 2024-05-09T03:44:32.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:32.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:32.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:32.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:32.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-09T03:44:32.492 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: monmap e2: 2 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} 2024-05-09T03:44:32.492 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: fsmap 2024-05-09T03:44:32.492 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: osdmap e4: 0 total, 0 up, 0 in 2024-05-09T03:44:32.492 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mgrmap e13: y(active, since 35s) 2024-05-09T03:44:32.492 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: overall HEALTH_OK 2024-05-09T03:44:32.492 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:32.492 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:32.492 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:32.492 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: Deploying daemon mon.b on smithi132 2024-05-09T03:44:32.493 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='client.? 172.21.15.132:0/2069902379' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-09T03:44:32.493 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:32.493 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:32.493 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:32.493 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-05-09T03:44:32.493 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 ceph-mon[23807]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-05-09T03:44:32.493 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 bash[23753]: a71bae52559df4ee8bff01c3c97e57148a5927605408fbf72ccd94fbceb27387 2024-05-09T03:44:32.493 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:32 smithi132 systemd[1]: Started Ceph mon.b for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:44:34.106 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-09T03:44:34.106 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-09T03:44:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:44:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: mon.a calling monitor election 2024-05-09T03:44:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: mon.c calling monitor election 2024-05-09T03:44:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: mon.b calling monitor election 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: fsmap 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: osdmap e4: 0 total, 0 up, 0 in 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: mgrmap e13: y(active, since 45s) 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: overall HEALTH_OK 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:37.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:37.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:37.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: mon.a calling monitor election 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: mon.c calling monitor election 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: mon.b calling monitor election 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-09T03:44:37.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} 2024-05-09T03:44:37.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: fsmap 2024-05-09T03:44:37.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: osdmap e4: 0 total, 0 up, 0 in 2024-05-09T03:44:37.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: mgrmap e13: y(active, since 45s) 2024-05-09T03:44:37.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: overall HEALTH_OK 2024-05-09T03:44:37.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:37.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:37.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:37.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:37.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:44:37.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: mon.a calling monitor election 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: mon.c calling monitor election 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: mon.b calling monitor election 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} 2024-05-09T03:44:37.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: fsmap 2024-05-09T03:44:37.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: osdmap e4: 0 total, 0 up, 0 in 2024-05-09T03:44:37.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: mgrmap e13: y(active, since 45s) 2024-05-09T03:44:37.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: overall HEALTH_OK 2024-05-09T03:44:37.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:37.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:37.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:37.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:38.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:38 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:38.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:38 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:38.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:38 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:38.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:38 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:38.710 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:38 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:38.710 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:38 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:38.710 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:38 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:38.710 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:38 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:38.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:38 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:38.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:38 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:38.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:38 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:44:38.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:38 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:38.881 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:44:38.881 INFO:teuthology.orchestra.run.smithi132.stdout:{"epoch":3,"fsid":"2777c79e-0db6-11ef-bc97-c7b262605968","modified":"2024-05-09T03:44:32.387541Z","created":"2024-05-09T03:42:55.620159Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3301","nonce":0},{"type":"v1","addr":"172.21.15.78:6790","nonce":0}]},"addr":"172.21.15.78:6790/0","public_addr":"172.21.15.78:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:3300","nonce":0},{"type":"v1","addr":"172.21.15.132:6789","nonce":0}]},"addr":"172.21.15.132:6789/0","public_addr":"172.21.15.132:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-05-09T03:44:38.883 INFO:teuthology.orchestra.run.smithi132.stderr:dumped monmap epoch 3 2024-05-09T03:44:39.481 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-05-09T03:44:39.481 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph config generate-minimal-conf 2024-05-09T03:44:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:39 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T03:44:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:39 smithi132 ceph-mon[23807]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:39 smithi132 ceph-mon[23807]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T03:44:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:39 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:39 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:39 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:39 smithi132 ceph-mon[23807]: Reconfiguring mon.a (unknown last config time)... 2024-05-09T03:44:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:39 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:39 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T03:44:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:39 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:39 smithi132 ceph-mon[23807]: Reconfiguring daemon mon.a on smithi078 2024-05-09T03:44:39.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:39 smithi132 ceph-mon[23807]: from='client.? 172.21.15.132:0/2249085832' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-09T03:44:39.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T03:44:39.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[27629]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:39.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T03:44:39.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:39.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:39.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:39.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[27629]: Reconfiguring mon.a (unknown last config time)... 2024-05-09T03:44:39.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:39.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T03:44:39.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:39.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[27629]: Reconfiguring daemon mon.a on smithi078 2024-05-09T03:44:39.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[27629]: from='client.? 172.21.15.132:0/2249085832' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-09T03:44:39.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T03:44:39.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[22905]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:39.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T03:44:39.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:39.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:39.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:39.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[22905]: Reconfiguring mon.a (unknown last config time)... 2024-05-09T03:44:39.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:39.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T03:44:39.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:39.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[22905]: Reconfiguring daemon mon.a on smithi078 2024-05-09T03:44:39.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:39 smithi078 ceph-mon[22905]: from='client.? 172.21.15.132:0/2249085832' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-09T03:44:41.002 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:41.002 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:40 smithi078 ceph-mon[22905]: Reconfiguring mon.c (monmap changed)... 2024-05-09T03:44:41.003 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:41.003 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T03:44:41.003 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:41.003 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:40 smithi078 ceph-mon[22905]: Reconfiguring daemon mon.c on smithi078 2024-05-09T03:44:41.003 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:41.003 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:40 smithi078 ceph-mon[27629]: Reconfiguring mon.c (monmap changed)... 2024-05-09T03:44:41.003 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:41.003 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T03:44:41.003 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:41.004 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:40 smithi078 ceph-mon[27629]: Reconfiguring daemon mon.c on smithi078 2024-05-09T03:44:41.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:41.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:40 smithi132 ceph-mon[23807]: Reconfiguring mon.c (monmap changed)... 2024-05-09T03:44:41.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:41.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T03:44:41.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:41.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:40 smithi132 ceph-mon[23807]: Reconfiguring daemon mon.c on smithi078 2024-05-09T03:44:41.343 INFO:teuthology.orchestra.run.smithi078.stdout:# minimal ceph.conf for 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:44:41.343 INFO:teuthology.orchestra.run.smithi078.stdout:[global] 2024-05-09T03:44:41.343 INFO:teuthology.orchestra.run.smithi078.stdout: fsid = 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:44:41.343 INFO:teuthology.orchestra.run.smithi078.stdout: mon_host = [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] [v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0] [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] 2024-05-09T03:44:42.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:41 smithi078 ceph-mon[22905]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:42.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:41 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3245999125' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:42.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:41 smithi078 ceph-mon[27629]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:42.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:41 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3245999125' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:42.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:41 smithi132 ceph-mon[23807]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:42.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:41 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3245999125' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:42.277 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-05-09T03:44:42.277 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-09T03:44:42.277 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.conf 2024-05-09T03:44:42.311 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-09T03:44:42.312 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:44:42.382 DEBUG:teuthology.orchestra.run.smithi132:> set -ex 2024-05-09T03:44:42.382 DEBUG:teuthology.orchestra.run.smithi132:> sudo dd of=/etc/ceph/ceph.conf 2024-05-09T03:44:42.414 DEBUG:teuthology.orchestra.run.smithi132:> set -ex 2024-05-09T03:44:42.414 DEBUG:teuthology.orchestra.run.smithi132:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:44:42.483 INFO:tasks.cephadm:Adding mgr.y on smithi078 2024-05-09T03:44:42.483 INFO:tasks.cephadm:Adding mgr.x on smithi132 2024-05-09T03:44:42.483 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch apply mgr '2;smithi078=y;smithi132=x' 2024-05-09T03:44:43.869 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:43 smithi132 ceph-mon[23807]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:43.869 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:43 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:43.869 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:43 smithi132 ceph-mon[23807]: Reconfiguring mon.b (monmap changed)... 2024-05-09T03:44:43.869 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:43 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:43.870 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:43 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T03:44:43.870 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:43 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:43.870 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:43 smithi132 ceph-mon[23807]: Reconfiguring daemon mon.b on smithi132 2024-05-09T03:44:44.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[22905]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:44.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:44.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[22905]: Reconfiguring mon.b (monmap changed)... 2024-05-09T03:44:44.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:44.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T03:44:44.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:44.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[22905]: Reconfiguring daemon mon.b on smithi132 2024-05-09T03:44:44.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[27629]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:44.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:44.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[27629]: Reconfiguring mon.b (monmap changed)... 2024-05-09T03:44:44.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T03:44:44.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T03:44:44.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:44.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:43 smithi078 ceph-mon[27629]: Reconfiguring daemon mon.b on smithi132 2024-05-09T03:44:45.383 INFO:teuthology.orchestra.run.smithi132.stdout:Scheduled mgr update... 2024-05-09T03:44:46.004 DEBUG:teuthology.orchestra.run.smithi132:mgr.x> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.x.service 2024-05-09T03:44:46.006 INFO:tasks.cephadm:Deploying OSDs... 2024-05-09T03:44:46.006 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-09T03:44:46.007 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/scratch_devs of=/dev/stdout 2024-05-09T03:44:46.027 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-09T03:44:46.027 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_1 2024-05-09T03:44:46.085 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-09T03:44:46.086 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-09T03:44:46.086 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-05-09T03:44:46.086 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-09T03:44:46.086 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-05-09T03:44:46.086 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-05-09 03:44:21.950108887 +0000 2024-05-09T03:44:46.086 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-05-09 03:42:05.867668065 +0000 2024-05-09T03:44:46.086 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-05-09 03:42:05.867668065 +0000 2024-05-09T03:44:46.086 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-05-09 03:42:05.867668065 +0000 2024-05-09T03:44:46.086 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-09T03:44:46.153 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-05-09T03:44:46.153 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-05-09T03:44:46.153 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000227477 s, 2.3 MB/s 2024-05-09T03:44:46.154 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-09T03:44:46.213 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_2 2024-05-09T03:44:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:46.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T03:44:46.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-09T03:44:46.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T03:44:46.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:46.271 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-09T03:44:46.271 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-09T03:44:46.271 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-05-09T03:44:46.272 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-09T03:44:46.272 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-05-09T03:44:46.272 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-05-09 03:44:21.950108887 +0000 2024-05-09T03:44:46.272 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-05-09 03:42:06.207663046 +0000 2024-05-09T03:44:46.272 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-05-09 03:42:06.207663046 +0000 2024-05-09T03:44:46.272 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-05-09 03:42:06.207663046 +0000 2024-05-09T03:44:46.272 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-09T03:44:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:46.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:46.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:46.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:46.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:46.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T03:44:46.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-09T03:44:46.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T03:44:46.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:46.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:46.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:46.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:46.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:46.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:46.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:46.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:46.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:46.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:46.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:46.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:46.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T03:44:46.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-09T03:44:46.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T03:44:46.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:46.339 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-05-09T03:44:46.339 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-05-09T03:44:46.339 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000184378 s, 2.8 MB/s 2024-05-09T03:44:46.340 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-09T03:44:46.397 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_3 2024-05-09T03:44:46.454 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-09T03:44:46.454 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-09T03:44:46.455 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-05-09T03:44:46.455 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-09T03:44:46.455 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-05-09T03:44:46.455 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-05-09 03:44:21.950108887 +0000 2024-05-09T03:44:46.455 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-05-09 03:42:06.538658160 +0000 2024-05-09T03:44:46.455 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-05-09 03:42:06.538658160 +0000 2024-05-09T03:44:46.455 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-05-09 03:42:06.538658160 +0000 2024-05-09T03:44:46.455 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-09T03:44:46.519 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-05-09T03:44:46.519 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-05-09T03:44:46.519 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000220247 s, 2.3 MB/s 2024-05-09T03:44:46.520 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-09T03:44:46.577 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_4 2024-05-09T03:44:46.633 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-09T03:44:46.634 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-09T03:44:46.634 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-05-09T03:44:46.634 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-09T03:44:46.634 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-05-09T03:44:46.634 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-05-09 03:44:21.951108872 +0000 2024-05-09T03:44:46.634 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-05-09 03:42:06.889652979 +0000 2024-05-09T03:44:46.634 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-05-09 03:42:06.889652979 +0000 2024-05-09T03:44:46.634 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-05-09 03:42:06.889652979 +0000 2024-05-09T03:44:46.634 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-09T03:44:46.697 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-05-09T03:44:46.697 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-05-09T03:44:46.698 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.0002266 s, 2.3 MB/s 2024-05-09T03:44:46.699 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-09T03:44:46.755 DEBUG:teuthology.orchestra.run.smithi132:> set -ex 2024-05-09T03:44:46.755 DEBUG:teuthology.orchestra.run.smithi132:> dd if=/scratch_devs of=/dev/stdout 2024-05-09T03:44:46.772 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-09T03:44:46.772 DEBUG:teuthology.orchestra.run.smithi132:> stat /dev/vg_nvme/lv_1 2024-05-09T03:44:46.827 INFO:teuthology.orchestra.run.smithi132.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-09T03:44:46.827 INFO:teuthology.orchestra.run.smithi132.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-09T03:44:46.827 INFO:teuthology.orchestra.run.smithi132.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-05-09T03:44:46.828 INFO:teuthology.orchestra.run.smithi132.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-09T03:44:46.828 INFO:teuthology.orchestra.run.smithi132.stdout:Context: system_u:object_r:device_t:s0 2024-05-09T03:44:46.828 INFO:teuthology.orchestra.run.smithi132.stdout:Access: 2024-05-09 03:44:31.229732447 +0000 2024-05-09T03:44:46.828 INFO:teuthology.orchestra.run.smithi132.stdout:Modify: 2024-05-09 03:42:07.730356418 +0000 2024-05-09T03:44:46.828 INFO:teuthology.orchestra.run.smithi132.stdout:Change: 2024-05-09 03:42:07.730356418 +0000 2024-05-09T03:44:46.828 INFO:teuthology.orchestra.run.smithi132.stdout: Birth: 2024-05-09 03:42:07.730356418 +0000 2024-05-09T03:44:46.828 DEBUG:teuthology.orchestra.run.smithi132:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-09T03:44:46.891 INFO:teuthology.orchestra.run.smithi132.stderr:1+0 records in 2024-05-09T03:44:46.892 INFO:teuthology.orchestra.run.smithi132.stderr:1+0 records out 2024-05-09T03:44:46.892 INFO:teuthology.orchestra.run.smithi132.stderr:512 bytes copied, 0.000247976 s, 2.1 MB/s 2024-05-09T03:44:46.893 DEBUG:teuthology.orchestra.run.smithi132:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-09T03:44:46.950 DEBUG:teuthology.orchestra.run.smithi132:> stat /dev/vg_nvme/lv_2 2024-05-09T03:44:47.005 INFO:teuthology.orchestra.run.smithi132.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-09T03:44:47.005 INFO:teuthology.orchestra.run.smithi132.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-09T03:44:47.005 INFO:teuthology.orchestra.run.smithi132.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-05-09T03:44:47.005 INFO:teuthology.orchestra.run.smithi132.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-09T03:44:47.006 INFO:teuthology.orchestra.run.smithi132.stdout:Context: system_u:object_r:device_t:s0 2024-05-09T03:44:47.006 INFO:teuthology.orchestra.run.smithi132.stdout:Access: 2024-05-09 03:44:31.230732421 +0000 2024-05-09T03:44:47.006 INFO:teuthology.orchestra.run.smithi132.stdout:Modify: 2024-05-09 03:42:08.095347086 +0000 2024-05-09T03:44:47.006 INFO:teuthology.orchestra.run.smithi132.stdout:Change: 2024-05-09 03:42:08.095347086 +0000 2024-05-09T03:44:47.006 INFO:teuthology.orchestra.run.smithi132.stdout: Birth: 2024-05-09 03:42:08.095347086 +0000 2024-05-09T03:44:47.006 DEBUG:teuthology.orchestra.run.smithi132:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-09T03:44:47.124 INFO:teuthology.orchestra.run.smithi132.stderr:1+0 records in 2024-05-09T03:44:47.124 INFO:teuthology.orchestra.run.smithi132.stderr:1+0 records out 2024-05-09T03:44:47.124 INFO:teuthology.orchestra.run.smithi132.stderr:512 bytes copied, 0.00020267 s, 2.5 MB/s 2024-05-09T03:44:47.125 DEBUG:teuthology.orchestra.run.smithi132:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-09T03:44:47.141 DEBUG:teuthology.orchestra.run.smithi132:> stat /dev/vg_nvme/lv_3 2024-05-09T03:44:47.199 INFO:teuthology.orchestra.run.smithi132.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-09T03:44:47.199 INFO:teuthology.orchestra.run.smithi132.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-09T03:44:47.199 INFO:teuthology.orchestra.run.smithi132.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-05-09T03:44:47.199 INFO:teuthology.orchestra.run.smithi132.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-09T03:44:47.199 INFO:teuthology.orchestra.run.smithi132.stdout:Context: system_u:object_r:device_t:s0 2024-05-09T03:44:47.199 INFO:teuthology.orchestra.run.smithi132.stdout:Access: 2024-05-09 03:44:47.082327216 +0000 2024-05-09T03:44:47.199 INFO:teuthology.orchestra.run.smithi132.stdout:Modify: 2024-05-09 03:42:08.432338469 +0000 2024-05-09T03:44:47.199 INFO:teuthology.orchestra.run.smithi132.stdout:Change: 2024-05-09 03:42:08.432338469 +0000 2024-05-09T03:44:47.200 INFO:teuthology.orchestra.run.smithi132.stdout: Birth: 2024-05-09 03:42:08.432338469 +0000 2024-05-09T03:44:47.200 DEBUG:teuthology.orchestra.run.smithi132:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-09T03:44:47.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:47 smithi132 ceph-mon[23807]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi078=y;smithi132=x", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:44:47.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:47 smithi132 ceph-mon[23807]: Saving service mgr spec with placement smithi078=y;smithi132=x;count:2 2024-05-09T03:44:47.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:47 smithi132 ceph-mon[23807]: Deploying daemon mgr.x on smithi132 2024-05-09T03:44:47.294 INFO:teuthology.orchestra.run.smithi132.stderr:1+0 records in 2024-05-09T03:44:47.294 INFO:teuthology.orchestra.run.smithi132.stderr:1+0 records out 2024-05-09T03:44:47.294 INFO:teuthology.orchestra.run.smithi132.stderr:512 bytes copied, 0.000192189 s, 2.7 MB/s 2024-05-09T03:44:47.295 DEBUG:teuthology.orchestra.run.smithi132:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-09T03:44:47.353 DEBUG:teuthology.orchestra.run.smithi132:> stat /dev/vg_nvme/lv_4 2024-05-09T03:44:47.411 INFO:teuthology.orchestra.run.smithi132.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-09T03:44:47.411 INFO:teuthology.orchestra.run.smithi132.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-09T03:44:47.411 INFO:teuthology.orchestra.run.smithi132.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-05-09T03:44:47.412 INFO:teuthology.orchestra.run.smithi132.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-09T03:44:47.412 INFO:teuthology.orchestra.run.smithi132.stdout:Context: system_u:object_r:device_t:s0 2024-05-09T03:44:47.412 INFO:teuthology.orchestra.run.smithi132.stdout:Access: 2024-05-09 03:44:47.255322794 +0000 2024-05-09T03:44:47.412 INFO:teuthology.orchestra.run.smithi132.stdout:Modify: 2024-05-09 03:42:08.813328728 +0000 2024-05-09T03:44:47.412 INFO:teuthology.orchestra.run.smithi132.stdout:Change: 2024-05-09 03:42:08.813328728 +0000 2024-05-09T03:44:47.412 INFO:teuthology.orchestra.run.smithi132.stdout: Birth: 2024-05-09 03:42:08.813328728 +0000 2024-05-09T03:44:47.413 DEBUG:teuthology.orchestra.run.smithi132:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-09T03:44:47.476 INFO:teuthology.orchestra.run.smithi132.stderr:1+0 records in 2024-05-09T03:44:47.476 INFO:teuthology.orchestra.run.smithi132.stderr:1+0 records out 2024-05-09T03:44:47.477 INFO:teuthology.orchestra.run.smithi132.stderr:512 bytes copied, 0.000193225 s, 2.6 MB/s 2024-05-09T03:44:47.478 DEBUG:teuthology.orchestra.run.smithi132:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-09T03:44:47.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:47 smithi078 ceph-mon[22905]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi078=y;smithi132=x", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:44:47.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:47 smithi078 ceph-mon[22905]: Saving service mgr spec with placement smithi078=y;smithi132=x;count:2 2024-05-09T03:44:47.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:47 smithi078 ceph-mon[22905]: Deploying daemon mgr.x on smithi132 2024-05-09T03:44:47.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:47 smithi078 ceph-mon[27629]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi078=y;smithi132=x", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:44:47.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:47 smithi078 ceph-mon[27629]: Saving service mgr spec with placement smithi078=y;smithi132=x;count:2 2024-05-09T03:44:47.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:47 smithi078 ceph-mon[27629]: Deploying daemon mgr.x on smithi132 2024-05-09T03:44:47.534 INFO:tasks.cephadm:Deploying osd.0 on smithi078 with /dev/vg_nvme/lv_4... 2024-05-09T03:44:47.534 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-05-09T03:44:48.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:48 smithi078 ceph-mon[22905]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:48.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:48 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:48.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:48 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:48.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:48 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:48.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:48 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:48.102 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:48 smithi078 ceph-mon[27629]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:48.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:48 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:48.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:48 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:48.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:48 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:48.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:48 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:48.118 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:48 smithi132 ceph-mon[23807]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:48.118 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:48 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:48.118 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:48 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:48.118 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:48 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:48.118 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:48 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:48.481 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:48 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:48.226+0000 7fc284090000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-09T03:44:48.753 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:48 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:48.555+0000 7fc284090000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-09T03:44:48.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:48 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:48.643+0000 7fc284090000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-09T03:44:50.171 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:44:50.190 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[22905]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:50.190 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:50.190 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:50.190 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[22905]: Reconfiguring mgr.y (unknown last config time)... 2024-05-09T03:44:50.190 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T03:44:50.190 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T03:44:50.190 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:50.190 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[22905]: Reconfiguring daemon mgr.y on smithi078 2024-05-09T03:44:50.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[27629]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:50.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:50.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:50.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[27629]: Reconfiguring mgr.y (unknown last config time)... 2024-05-09T03:44:50.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T03:44:50.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T03:44:50.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:50.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:49 smithi078 ceph-mon[27629]: Reconfiguring daemon mgr.y on smithi078 2024-05-09T03:44:50.192 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_4 2024-05-09T03:44:50.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:49 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:49.796+0000 7fc284090000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-09T03:44:50.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:49 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:49.962+0000 7fc284090000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-09T03:44:50.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:50.042+0000 7fc284090000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-09T03:44:50.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:49 smithi132 ceph-mon[23807]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:50.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:49 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:50.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:49 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:50.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:49 smithi132 ceph-mon[23807]: Reconfiguring mgr.y (unknown last config time)... 2024-05-09T03:44:50.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:49 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T03:44:50.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:49 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T03:44:50.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:49 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:50.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:49 smithi132 ceph-mon[23807]: Reconfiguring daemon mgr.y on smithi078 2024-05-09T03:44:50.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:50.356+0000 7fc284090000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-09T03:44:50.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:50.445+0000 7fc284090000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-09T03:44:50.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:50.536+0000 7fc284090000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-09T03:44:51.219 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:50.778+0000 7fc284090000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:44:51.503 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:51.218+0000 7fc284090000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-09T03:44:52.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:51.706+0000 7fc284090000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-09T03:44:52.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:51.798+0000 7fc284090000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-09T03:44:52.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:51 smithi132 ceph-mon[23807]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:52.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:51 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:52.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:51 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:52.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:51 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:52.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:51 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:52.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:51 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:52.119 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:51 smithi078 ceph-mon[27629]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:52.119 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:51 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:52.119 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:51 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:52.119 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:51 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:52.119 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:51 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:52.119 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:51 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:52.120 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:51 smithi078 ceph-mon[22905]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:52.120 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:51 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:52.120 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:51 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:44:52.120 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:51 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:52.120 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:51 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:44:52.120 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:51 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:52.503 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:52 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:52.092+0000 7fc284090000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-09T03:44:52.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:52 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:52.179+0000 7fc284090000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-09T03:44:52.926 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:52 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:52.590+0000 7fc284090000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-09T03:44:52.926 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:52 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:52.720+0000 7fc284090000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-09T03:44:52.926 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:52 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:52.812+0000 7fc284090000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-09T03:44:52.926 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:52 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:52.926+0000 7fc284090000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-09T03:44:53.138 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:44:53.138 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:44:53.138 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:53.138 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:44:53.138 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:44:53.138 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:53.138 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:44:53.138 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:44:53.138 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:53.139 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:44:53.139 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:44:53.139 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:53.253 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:53.016+0000 7fc284090000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-09T03:44:53.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:44:53.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:44:53.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:44:53.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:44:53.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:44:53.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:44:53.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:53.555+0000 7fc284090000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-09T03:44:54.163 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[22905]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:54.163 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[22905]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:44:54.163 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[22905]: Standby manager daemon x started 2024-05-09T03:44:54.163 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/3111361041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T03:44:54.163 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/3111361041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T03:44:54.163 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/3111361041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T03:44:54.163 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/3111361041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T03:44:54.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[27629]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:54.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[27629]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:44:54.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[27629]: Standby manager daemon x started 2024-05-09T03:44:54.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/3111361041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T03:44:54.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/3111361041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T03:44:54.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/3111361041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T03:44:54.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:53 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/3111361041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T03:44:54.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:44:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:44:53.917+0000 7fc284090000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-09T03:44:54.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:53 smithi132 ceph-mon[23807]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:54.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:53 smithi132 ceph-mon[23807]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:44:54.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:53 smithi132 ceph-mon[23807]: Standby manager daemon x started 2024-05-09T03:44:54.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:53 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/3111361041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T03:44:54.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:53 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/3111361041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T03:44:54.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:53 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/3111361041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T03:44:54.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:53 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/3111361041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T03:44:55.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:54 smithi132 ceph-mon[23807]: mgrmap e14: y(active, since 61s), standbys: x 2024-05-09T03:44:55.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:54 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T03:44:55.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:54 smithi078 ceph-mon[22905]: mgrmap e14: y(active, since 61s), standbys: x 2024-05-09T03:44:55.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:54 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T03:44:55.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:54 smithi078 ceph-mon[27629]: mgrmap e14: y(active, since 61s), standbys: x 2024-05-09T03:44:55.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:54 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T03:44:56.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:55 smithi132 ceph-mon[23807]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:56.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:55 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2293201831' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbcf6ecf-5076-4a63-b6f7-df5e34185ddd"}]: dispatch 2024-05-09T03:44:56.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:55 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2293201831' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbcf6ecf-5076-4a63-b6f7-df5e34185ddd"}]': finished 2024-05-09T03:44:56.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:55 smithi132 ceph-mon[23807]: osdmap e5: 1 total, 0 up, 1 in 2024-05-09T03:44:56.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:55 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:44:56.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:55 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/62870916' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:44:56.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:55 smithi078 ceph-mon[22905]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:56.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:55 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2293201831' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbcf6ecf-5076-4a63-b6f7-df5e34185ddd"}]: dispatch 2024-05-09T03:44:56.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:55 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2293201831' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbcf6ecf-5076-4a63-b6f7-df5e34185ddd"}]': finished 2024-05-09T03:44:56.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:55 smithi078 ceph-mon[22905]: osdmap e5: 1 total, 0 up, 1 in 2024-05-09T03:44:56.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:55 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:44:56.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:55 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/62870916' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:44:56.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:55 smithi078 ceph-mon[27629]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:56.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:55 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2293201831' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbcf6ecf-5076-4a63-b6f7-df5e34185ddd"}]: dispatch 2024-05-09T03:44:56.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:55 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2293201831' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbcf6ecf-5076-4a63-b6f7-df5e34185ddd"}]': finished 2024-05-09T03:44:56.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:55 smithi078 ceph-mon[27629]: osdmap e5: 1 total, 0 up, 1 in 2024-05-09T03:44:56.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:55 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:44:56.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:55 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/62870916' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:44:58.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:57 smithi078 ceph-mon[22905]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:58.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:57 smithi078 ceph-mon[27629]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:44:58.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:57 smithi132 ceph-mon[23807]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:00.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:44:59 smithi132 ceph-mon[23807]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:00.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:44:59 smithi078 ceph-mon[22905]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:00.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:44:59 smithi078 ceph-mon[27629]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:02.231 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:01 smithi078 ceph-mon[27629]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:02.231 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:01 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-09T03:45:02.231 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:01 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:02.231 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:01 smithi078 ceph-mon[27629]: Deploying daemon osd.0 on smithi078 2024-05-09T03:45:02.232 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:01 smithi078 ceph-mon[22905]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:02.232 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:01 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-09T03:45:02.232 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:01 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:02.232 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:01 smithi078 ceph-mon[22905]: Deploying daemon osd.0 on smithi078 2024-05-09T03:45:02.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:01 smithi132 ceph-mon[23807]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:02.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:01 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-09T03:45:02.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:01 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:02.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:01 smithi132 ceph-mon[23807]: Deploying daemon osd.0 on smithi078 2024-05-09T03:45:04.121 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:03 smithi078 ceph-mon[22905]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:04.121 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:03 smithi078 ceph-mon[27629]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:04.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:03 smithi132 ceph-mon[23807]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:06.225 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:05 smithi078 ceph-mon[27629]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:06.225 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:06.226 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:06.226 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:06.226 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:06.226 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:05 smithi078 ceph-mon[22905]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:06.226 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:06.226 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:06.226 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:06.226 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:06.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:05 smithi132 ceph-mon[23807]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:06.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:06.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:06.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:06.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:08.119 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:07 smithi078 ceph-mon[27629]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:08.119 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:07 smithi078 ceph-mon[22905]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:08.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:07 smithi132 ceph-mon[23807]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:09.104 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 0 on host 'smithi078' 2024-05-09T03:45:09.466 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:09.466 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:09.466 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[22905]: from='osd.0 [v2:172.21.15.78:6802/3700260173,v1:172.21.15.78:6803/3700260173]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-09T03:45:09.466 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[22905]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-09T03:45:09.466 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:09.466 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:09.467 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:09.467 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:09.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:09.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:09.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[27629]: from='osd.0 [v2:172.21.15.78:6802/3700260173,v1:172.21.15.78:6803/3700260173]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-09T03:45:09.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[27629]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-09T03:45:09.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:09.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:09.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:09.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:09 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:09.741 DEBUG:teuthology.orchestra.run.smithi078:osd.0> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.0.service 2024-05-09T03:45:09.743 INFO:tasks.cephadm:Deploying osd.1 on smithi078 with /dev/vg_nvme/lv_3... 2024-05-09T03:45:09.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-05-09T03:45:09.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:09 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:09 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:09 smithi132 ceph-mon[23807]: from='osd.0 [v2:172.21.15.78:6802/3700260173,v1:172.21.15.78:6803/3700260173]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-09T03:45:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:09 smithi132 ceph-mon[23807]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-09T03:45:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:09 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:09 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:09 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:09 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:10.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:10 smithi078 ceph-mon[22905]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:10 smithi078 ceph-mon[22905]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-09T03:45:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:10 smithi078 ceph-mon[22905]: osdmap e6: 1 total, 0 up, 1 in 2024-05-09T03:45:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:10 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:45:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:10 smithi078 ceph-mon[22905]: from='osd.0 [v2:172.21.15.78:6802/3700260173,v1:172.21.15.78:6803/3700260173]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:10 smithi078 ceph-mon[22905]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:10 smithi078 ceph-mon[27629]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:10 smithi078 ceph-mon[27629]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-09T03:45:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:10 smithi078 ceph-mon[27629]: osdmap e6: 1 total, 0 up, 1 in 2024-05-09T03:45:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:10 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:45:10.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:10 smithi078 ceph-mon[27629]: from='osd.0 [v2:172.21.15.78:6802/3700260173,v1:172.21.15.78:6803/3700260173]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:10.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:10 smithi078 ceph-mon[27629]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:10 smithi132 ceph-mon[23807]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:10 smithi132 ceph-mon[23807]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-09T03:45:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:10 smithi132 ceph-mon[23807]: osdmap e6: 1 total, 0 up, 1 in 2024-05-09T03:45:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:10 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:45:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:10 smithi132 ceph-mon[23807]: from='osd.0 [v2:172.21.15.78:6802/3700260173,v1:172.21.15.78:6803/3700260173]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:10 smithi132 ceph-mon[23807]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:10.917 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 03:45:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0[30489]: 2024-05-09T03:45:10.586+0000 7f0da90a7700 -1 osd.0 0 waiting for initial osdmap 2024-05-09T03:45:10.917 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 03:45:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0[30489]: 2024-05-09T03:45:10.588+0000 7f0da3a3e700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T03:45:11.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:11 smithi078 ceph-mon[22905]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:11.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:11 smithi078 ceph-mon[22905]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-09T03:45:11.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:11 smithi078 ceph-mon[22905]: osdmap e7: 1 total, 0 up, 1 in 2024-05-09T03:45:11.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:11 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:45:11.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:11 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:45:11.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:11 smithi078 ceph-mon[27629]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:11.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:11 smithi078 ceph-mon[27629]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-09T03:45:11.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:11 smithi078 ceph-mon[27629]: osdmap e7: 1 total, 0 up, 1 in 2024-05-09T03:45:11.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:11 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:45:11.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:11 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:45:12.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:11 smithi132 ceph-mon[23807]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:12.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:11 smithi132 ceph-mon[23807]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-09T03:45:12.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:11 smithi132 ceph-mon[23807]: osdmap e7: 1 total, 0 up, 1 in 2024-05-09T03:45:12.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:11 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:45:12.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:11 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:45:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:12 smithi078 ceph-mon[22905]: purged_snaps scrub starts 2024-05-09T03:45:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:12 smithi078 ceph-mon[22905]: purged_snaps scrub ok 2024-05-09T03:45:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:12 smithi078 ceph-mon[22905]: osd.0 [v2:172.21.15.78:6802/3700260173,v1:172.21.15.78:6803/3700260173] boot 2024-05-09T03:45:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:12 smithi078 ceph-mon[22905]: osdmap e8: 1 total, 1 up, 1 in 2024-05-09T03:45:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:12 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:45:12.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:12 smithi078 ceph-mon[27629]: purged_snaps scrub starts 2024-05-09T03:45:12.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:12 smithi078 ceph-mon[27629]: purged_snaps scrub ok 2024-05-09T03:45:12.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:12 smithi078 ceph-mon[27629]: osd.0 [v2:172.21.15.78:6802/3700260173,v1:172.21.15.78:6803/3700260173] boot 2024-05-09T03:45:12.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:12 smithi078 ceph-mon[27629]: osdmap e8: 1 total, 1 up, 1 in 2024-05-09T03:45:12.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:12 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:45:13.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:12 smithi132 ceph-mon[23807]: purged_snaps scrub starts 2024-05-09T03:45:13.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:12 smithi132 ceph-mon[23807]: purged_snaps scrub ok 2024-05-09T03:45:13.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:12 smithi132 ceph-mon[23807]: osd.0 [v2:172.21.15.78:6802/3700260173,v1:172.21.15.78:6803/3700260173] boot 2024-05-09T03:45:13.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:12 smithi132 ceph-mon[23807]: osdmap e8: 1 total, 1 up, 1 in 2024-05-09T03:45:13.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:12 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:45:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[27629]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[27629]: osdmap e9: 1 total, 1 up, 1 in 2024-05-09T03:45:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[27629]: Detected new or changed devices on smithi078 2024-05-09T03:45:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:45:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[27629]: Adjusting osd_memory_target on smithi078 to 16064M 2024-05-09T03:45:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:13.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[22905]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:13.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[22905]: osdmap e9: 1 total, 1 up, 1 in 2024-05-09T03:45:13.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[22905]: Detected new or changed devices on smithi078 2024-05-09T03:45:13.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:13.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:45:13.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[22905]: Adjusting osd_memory_target on smithi078 to 16064M 2024-05-09T03:45:13.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:13.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:13 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:13 smithi132 ceph-mon[23807]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-09T03:45:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:13 smithi132 ceph-mon[23807]: osdmap e9: 1 total, 1 up, 1 in 2024-05-09T03:45:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:13 smithi132 ceph-mon[23807]: Detected new or changed devices on smithi078 2024-05-09T03:45:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:13 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:13 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:45:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:13 smithi132 ceph-mon[23807]: Adjusting osd_memory_target on smithi078 to 16064M 2024-05-09T03:45:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:13 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:13 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:14.527 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:45:14.547 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_3 2024-05-09T03:45:15.794 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:15 smithi078 ceph-mon[22905]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:15.794 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:15 smithi078 ceph-mon[27629]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:16.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:15 smithi132 ceph-mon[23807]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:16.743 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:16 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:45:16.743 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:16 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:45:16.743 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:16 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:16.743 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:16 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:45:16.743 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:16 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:45:16.743 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:16 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:16 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:45:17.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:16 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:45:17.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:16 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:17.720 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:17 smithi078 ceph-mon[22905]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:45:17.720 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:17 smithi078 ceph-mon[22905]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:17.720 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:17 smithi078 ceph-mon[27629]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:45:17.720 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:17 smithi078 ceph-mon[27629]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:18.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:17 smithi132 ceph-mon[23807]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:45:18.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:17 smithi132 ceph-mon[23807]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:19.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:18 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/992929396' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "25c3b194-8ee3-41d3-a172-90c263918e62"}]: dispatch 2024-05-09T03:45:19.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:18 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/992929396' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "25c3b194-8ee3-41d3-a172-90c263918e62"}]': finished 2024-05-09T03:45:19.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:18 smithi132 ceph-mon[23807]: osdmap e10: 2 total, 1 up, 2 in 2024-05-09T03:45:19.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:18 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:19.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:18 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/992929396' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "25c3b194-8ee3-41d3-a172-90c263918e62"}]: dispatch 2024-05-09T03:45:19.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:18 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/992929396' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "25c3b194-8ee3-41d3-a172-90c263918e62"}]': finished 2024-05-09T03:45:19.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:18 smithi078 ceph-mon[22905]: osdmap e10: 2 total, 1 up, 2 in 2024-05-09T03:45:19.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:18 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:19.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:18 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/992929396' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "25c3b194-8ee3-41d3-a172-90c263918e62"}]: dispatch 2024-05-09T03:45:19.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:18 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/992929396' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "25c3b194-8ee3-41d3-a172-90c263918e62"}]': finished 2024-05-09T03:45:19.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:18 smithi078 ceph-mon[27629]: osdmap e10: 2 total, 1 up, 2 in 2024-05-09T03:45:19.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:18 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:20.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:19 smithi132 ceph-mon[23807]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:20.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:19 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/253327178' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:45:20.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:19 smithi078 ceph-mon[22905]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:20.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:19 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/253327178' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:45:20.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:19 smithi078 ceph-mon[27629]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:20.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:19 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/253327178' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:45:22.136 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:21 smithi078 ceph-mon[22905]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:22.137 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:21 smithi078 ceph-mon[27629]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:22.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:21 smithi132 ceph-mon[23807]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:24.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:23 smithi078 ceph-mon[22905]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:24.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:23 smithi078 ceph-mon[27629]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:24.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:23 smithi132 ceph-mon[23807]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:26.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:25 smithi132 ceph-mon[23807]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:26.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-09T03:45:26.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:26.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:25 smithi132 ceph-mon[23807]: Deploying daemon osd.1 on smithi078 2024-05-09T03:45:26.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:25 smithi078 ceph-mon[22905]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:26.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-09T03:45:26.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:26.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:25 smithi078 ceph-mon[22905]: Deploying daemon osd.1 on smithi078 2024-05-09T03:45:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:25 smithi078 ceph-mon[27629]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-09T03:45:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:25 smithi078 ceph-mon[27629]: Deploying daemon osd.1 on smithi078 2024-05-09T03:45:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:27 smithi078 ceph-mon[22905]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:27 smithi078 ceph-mon[27629]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:28.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:27 smithi132 ceph-mon[23807]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:30.184 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:29 smithi078 ceph-mon[22905]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:30.184 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:29 smithi078 ceph-mon[27629]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:30.204 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:29 smithi132 ceph-mon[23807]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:31.489 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:31 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:31.489 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:31 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:31.489 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:31 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:31.489 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:31 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:31.489 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:31 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:31.490 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:31 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:31.490 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:31 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:31.490 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:31 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:31.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:31 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:31.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:31 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:31.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:31 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:31.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:31 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:32.419 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:32 smithi132 ceph-mon[23807]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:32.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:32 smithi078 ceph-mon[22905]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:32 smithi078 ceph-mon[27629]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:33.085 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 1 on host 'smithi078' 2024-05-09T03:45:33.639 DEBUG:teuthology.orchestra.run.smithi078:osd.1> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.1.service 2024-05-09T03:45:33.641 INFO:tasks.cephadm:Deploying osd.2 on smithi078 with /dev/vg_nvme/lv_2... 2024-05-09T03:45:33.642 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-05-09T03:45:34.243 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:34 smithi078 ceph-mon[22905]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:34.243 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:34 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:34.243 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:34 smithi078 ceph-mon[22905]: from='osd.1 [v2:172.21.15.78:6810/2683958178,v1:172.21.15.78:6811/2683958178]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-09T03:45:34.243 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:34 smithi078 ceph-mon[22905]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-09T03:45:34.243 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:34 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:34.243 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:34 smithi078 ceph-mon[27629]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:34.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:34 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:34.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:34 smithi078 ceph-mon[27629]: from='osd.1 [v2:172.21.15.78:6810/2683958178,v1:172.21.15.78:6811/2683958178]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-09T03:45:34.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:34 smithi078 ceph-mon[27629]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-09T03:45:34.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:34 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:34.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:34 smithi132 ceph-mon[23807]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:34.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:34 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:34.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:34 smithi132 ceph-mon[23807]: from='osd.1 [v2:172.21.15.78:6810/2683958178,v1:172.21.15.78:6811/2683958178]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-09T03:45:34.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:34 smithi132 ceph-mon[23807]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-09T03:45:34.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:34 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:35.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:35 smithi078 ceph-mon[22905]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-09T03:45:35.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:35 smithi078 ceph-mon[22905]: osdmap e11: 2 total, 1 up, 2 in 2024-05-09T03:45:35.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:35 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:35.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:35 smithi078 ceph-mon[22905]: from='osd.1 [v2:172.21.15.78:6810/2683958178,v1:172.21.15.78:6811/2683958178]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:35.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:35 smithi078 ceph-mon[22905]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:35.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:35 smithi078 ceph-mon[27629]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-09T03:45:35.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:35 smithi078 ceph-mon[27629]: osdmap e11: 2 total, 1 up, 2 in 2024-05-09T03:45:35.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:35 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:35.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:35 smithi078 ceph-mon[27629]: from='osd.1 [v2:172.21.15.78:6810/2683958178,v1:172.21.15.78:6811/2683958178]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:35.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:35 smithi078 ceph-mon[27629]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:35.248 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 03:45:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1[33457]: 2024-05-09T03:45:35.176+0000 7f3852d05700 -1 osd.1 0 waiting for initial osdmap 2024-05-09T03:45:35.248 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 03:45:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1[33457]: 2024-05-09T03:45:35.179+0000 7f384be99700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T03:45:35.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:35 smithi132 ceph-mon[23807]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-09T03:45:35.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:35 smithi132 ceph-mon[23807]: osdmap e11: 2 total, 1 up, 2 in 2024-05-09T03:45:35.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:35 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:35.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:35 smithi132 ceph-mon[23807]: from='osd.1 [v2:172.21.15.78:6810/2683958178,v1:172.21.15.78:6811/2683958178]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:35.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:35 smithi132 ceph-mon[23807]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:36.497 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:36 smithi078 ceph-mon[27629]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:36.497 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:36 smithi078 ceph-mon[27629]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-09T03:45:36.497 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:36 smithi078 ceph-mon[27629]: osdmap e12: 2 total, 1 up, 2 in 2024-05-09T03:45:36.497 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:36 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:36.497 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:36 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:36.497 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:36 smithi078 ceph-mon[22905]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:36.497 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:36 smithi078 ceph-mon[22905]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-09T03:45:36.498 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:36 smithi078 ceph-mon[22905]: osdmap e12: 2 total, 1 up, 2 in 2024-05-09T03:45:36.498 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:36 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:36.498 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:36 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:36 smithi132 ceph-mon[23807]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:36 smithi132 ceph-mon[23807]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-09T03:45:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:36 smithi132 ceph-mon[23807]: osdmap e12: 2 total, 1 up, 2 in 2024-05-09T03:45:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:36 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:36 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:37.409 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:37 smithi078 ceph-mon[27629]: purged_snaps scrub starts 2024-05-09T03:45:37.409 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:37 smithi078 ceph-mon[27629]: purged_snaps scrub ok 2024-05-09T03:45:37.409 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:37 smithi078 ceph-mon[27629]: osd.1 [v2:172.21.15.78:6810/2683958178,v1:172.21.15.78:6811/2683958178] boot 2024-05-09T03:45:37.409 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:37 smithi078 ceph-mon[27629]: osdmap e13: 2 total, 2 up, 2 in 2024-05-09T03:45:37.409 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:37 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:37.410 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:37 smithi078 ceph-mon[22905]: purged_snaps scrub starts 2024-05-09T03:45:37.410 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:37 smithi078 ceph-mon[22905]: purged_snaps scrub ok 2024-05-09T03:45:37.410 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:37 smithi078 ceph-mon[22905]: osd.1 [v2:172.21.15.78:6810/2683958178,v1:172.21.15.78:6811/2683958178] boot 2024-05-09T03:45:37.410 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:37 smithi078 ceph-mon[22905]: osdmap e13: 2 total, 2 up, 2 in 2024-05-09T03:45:37.410 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:37 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:37.414 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:45:37.432 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_2 2024-05-09T03:45:37.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:37 smithi132 ceph-mon[23807]: purged_snaps scrub starts 2024-05-09T03:45:37.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:37 smithi132 ceph-mon[23807]: purged_snaps scrub ok 2024-05-09T03:45:37.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:37 smithi132 ceph-mon[23807]: osd.1 [v2:172.21.15.78:6810/2683958178,v1:172.21.15.78:6811/2683958178] boot 2024-05-09T03:45:37.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:37 smithi132 ceph-mon[23807]: osdmap e13: 2 total, 2 up, 2 in 2024-05-09T03:45:37.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:37 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:45:38.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:38 smithi078 ceph-mon[27629]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:38.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:38 smithi078 ceph-mon[27629]: osdmap e14: 2 total, 2 up, 2 in 2024-05-09T03:45:38.365 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:38 smithi078 ceph-mon[22905]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:38.365 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:38 smithi078 ceph-mon[22905]: osdmap e14: 2 total, 2 up, 2 in 2024-05-09T03:45:38.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:38 smithi132 ceph-mon[23807]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-09T03:45:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:38 smithi132 ceph-mon[23807]: osdmap e14: 2 total, 2 up, 2 in 2024-05-09T03:45:40.283 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[22905]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:40.283 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[22905]: Detected new or changed devices on smithi078 2024-05-09T03:45:40.283 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:40.283 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:45:40.283 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:45:40.283 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[22905]: Adjusting osd_memory_target on smithi078 to 8032M 2024-05-09T03:45:40.283 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:40.284 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:40.284 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:40.284 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:40.284 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:40.284 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:40.284 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[27629]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:40.284 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[27629]: Detected new or changed devices on smithi078 2024-05-09T03:45:40.284 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:40.284 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:45:40.284 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:45:40.284 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[27629]: Adjusting osd_memory_target on smithi078 to 8032M 2024-05-09T03:45:40.285 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:40.285 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:40.285 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:40.285 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:40.285 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:40.285 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:40.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:40 smithi132 ceph-mon[23807]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:40.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:40 smithi132 ceph-mon[23807]: Detected new or changed devices on smithi078 2024-05-09T03:45:40.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:40.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:45:40.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:45:40.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:40 smithi132 ceph-mon[23807]: Adjusting osd_memory_target on smithi078 to 8032M 2024-05-09T03:45:40.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:40.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:40.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:40.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:40.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:40.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:41.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:41 smithi132 ceph-mon[23807]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:45:41.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:41 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:45:41.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:41 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:45:41.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:41 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:41.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:41 smithi078 ceph-mon[22905]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:45:41.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:41 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:45:41.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:41 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:45:41.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:41 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:41.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:41 smithi078 ceph-mon[27629]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:45:41.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:41 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:45:41.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:41 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:45:41.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:41 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:42.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:42 smithi132 ceph-mon[23807]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:42.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:42 smithi078 ceph-mon[22905]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:42.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:42 smithi078 ceph-mon[27629]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:43 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1031671899' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86841bbd-3c86-4114-9347-e1b9dbd8358f"}]: dispatch 2024-05-09T03:45:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:43 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1031671899' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86841bbd-3c86-4114-9347-e1b9dbd8358f"}]': finished 2024-05-09T03:45:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:43 smithi132 ceph-mon[23807]: osdmap e15: 3 total, 2 up, 3 in 2024-05-09T03:45:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:43 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:45:43.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:43 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1031671899' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86841bbd-3c86-4114-9347-e1b9dbd8358f"}]: dispatch 2024-05-09T03:45:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:43 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1031671899' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86841bbd-3c86-4114-9347-e1b9dbd8358f"}]': finished 2024-05-09T03:45:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:43 smithi078 ceph-mon[27629]: osdmap e15: 3 total, 2 up, 3 in 2024-05-09T03:45:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:43 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:45:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:43 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1031671899' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86841bbd-3c86-4114-9347-e1b9dbd8358f"}]: dispatch 2024-05-09T03:45:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:43 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1031671899' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86841bbd-3c86-4114-9347-e1b9dbd8358f"}]': finished 2024-05-09T03:45:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:43 smithi078 ceph-mon[22905]: osdmap e15: 3 total, 2 up, 3 in 2024-05-09T03:45:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:43 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:45:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:44 smithi132 ceph-mon[23807]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:44 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2542963857' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:45:44.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:44 smithi078 ceph-mon[22905]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:44.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:44 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2542963857' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:45:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:44 smithi078 ceph-mon[27629]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:44 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2542963857' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:45:46.434 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:46 smithi078 ceph-mon[22905]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:46.434 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:46 smithi078 ceph-mon[27629]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:46.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:46 smithi132 ceph-mon[23807]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:48.418 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:48 smithi078 ceph-mon[22905]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:48.418 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:48 smithi078 ceph-mon[27629]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:48 smithi132 ceph-mon[23807]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:49.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:49 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-09T03:45:49.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:49 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:49.363 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:49 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-09T03:45:49.363 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:49 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:49.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:49 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-09T03:45:49.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:49 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:50.305 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:50 smithi078 ceph-mon[22905]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:50.305 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:50 smithi078 ceph-mon[22905]: Deploying daemon osd.2 on smithi078 2024-05-09T03:45:50.305 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:50 smithi078 ceph-mon[27629]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:50.305 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:50 smithi078 ceph-mon[27629]: Deploying daemon osd.2 on smithi078 2024-05-09T03:45:50.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:50 smithi132 ceph-mon[23807]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:50 smithi132 ceph-mon[23807]: Deploying daemon osd.2 on smithi078 2024-05-09T03:45:52.467 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:52 smithi078 ceph-mon[22905]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:52.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:52 smithi078 ceph-mon[27629]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:52.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:52 smithi132 ceph-mon[23807]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:53.388 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:45:53.388 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:45:53.388 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:53 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:45:53.388 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:53 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:45:53.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:53 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:45:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:53 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:45:54.369 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:54 smithi078 ceph-mon[22905]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:54.369 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:54 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:54.369 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:54 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:54.369 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:54 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:54.369 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:54 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:54.370 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:54 smithi078 ceph-mon[27629]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:54.370 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:54 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:54.370 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:54 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:54.370 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:54 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:54.370 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:54 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:54.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:54 smithi132 ceph-mon[23807]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:54.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:54 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:54.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:54 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:45:54.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:54 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:45:54.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:54 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:45:56.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:56 smithi132 ceph-mon[23807]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:56.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:56 smithi078 ceph-mon[22905]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:56.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:56 smithi078 ceph-mon[27629]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:56.685 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 2 on host 'smithi078' 2024-05-09T03:45:57.275 DEBUG:teuthology.orchestra.run.smithi078:osd.2> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.2.service 2024-05-09T03:45:57.277 INFO:tasks.cephadm:Deploying osd.3 on smithi078 with /dev/vg_nvme/lv_1... 2024-05-09T03:45:57.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-05-09T03:45:57.946 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:57 smithi078 ceph-mon[22905]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:57.946 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:57 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:57.946 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:57 smithi078 ceph-mon[22905]: from='osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-09T03:45:57.947 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:57 smithi078 ceph-mon[27629]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:57.947 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:57 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:57.947 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:57 smithi078 ceph-mon[27629]: from='osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-09T03:45:58.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:57 smithi132 ceph-mon[23807]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:45:58.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:57 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:45:58.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:57 smithi132 ceph-mon[23807]: from='osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-09T03:45:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:58 smithi132 ceph-mon[23807]: from='osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-09T03:45:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:58 smithi132 ceph-mon[23807]: osdmap e16: 3 total, 2 up, 3 in 2024-05-09T03:45:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:58 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:45:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:58 smithi132 ceph-mon[23807]: from='osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:59.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:58 smithi078 ceph-mon[27629]: from='osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-09T03:45:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:58 smithi078 ceph-mon[27629]: osdmap e16: 3 total, 2 up, 3 in 2024-05-09T03:45:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:58 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:45:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:58 smithi078 ceph-mon[27629]: from='osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:59.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:58 smithi078 ceph-mon[22905]: from='osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-09T03:45:59.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:58 smithi078 ceph-mon[22905]: osdmap e16: 3 total, 2 up, 3 in 2024-05-09T03:45:59.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:58 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:45:59.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:58 smithi078 ceph-mon[22905]: from='osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:45:59.012 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 03:45:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2[36375]: 2024-05-09T03:45:58.703+0000 7ffb36446700 -1 osd.2 0 waiting for initial osdmap 2024-05-09T03:45:59.012 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 03:45:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2[36375]: 2024-05-09T03:45:58.709+0000 7ffb2fddb700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T03:46:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:59 smithi132 ceph-mon[23807]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:46:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:59 smithi132 ceph-mon[23807]: from='osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-09T03:46:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:59 smithi132 ceph-mon[23807]: osdmap e17: 3 total, 2 up, 3 in 2024-05-09T03:46:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:59 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:46:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:59 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:46:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:45:59 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:00.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:59 smithi078 ceph-mon[22905]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:46:00.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:59 smithi078 ceph-mon[22905]: from='osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-09T03:46:00.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:59 smithi078 ceph-mon[22905]: osdmap e17: 3 total, 2 up, 3 in 2024-05-09T03:46:00.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:59 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:46:00.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:59 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:46:00.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:45:59 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:00.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:59 smithi078 ceph-mon[27629]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:46:00.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:59 smithi078 ceph-mon[27629]: from='osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-09T03:46:00.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:59 smithi078 ceph-mon[27629]: osdmap e17: 3 total, 2 up, 3 in 2024-05-09T03:46:00.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:59 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:46:00.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:59 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:46:00.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:45:59 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:00.053 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:46:00.072 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_1 2024-05-09T03:46:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:00 smithi132 ceph-mon[23807]: purged_snaps scrub starts 2024-05-09T03:46:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:00 smithi132 ceph-mon[23807]: purged_snaps scrub ok 2024-05-09T03:46:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:00 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:46:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:00 smithi132 ceph-mon[23807]: osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399] boot 2024-05-09T03:46:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:00 smithi132 ceph-mon[23807]: osdmap e18: 3 total, 3 up, 3 in 2024-05-09T03:46:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:00 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:46:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:00 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-09T03:46:01.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[22905]: purged_snaps scrub starts 2024-05-09T03:46:01.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[22905]: purged_snaps scrub ok 2024-05-09T03:46:01.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:46:01.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[22905]: osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399] boot 2024-05-09T03:46:01.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[22905]: osdmap e18: 3 total, 3 up, 3 in 2024-05-09T03:46:01.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:46:01.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-09T03:46:01.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[27629]: purged_snaps scrub starts 2024-05-09T03:46:01.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[27629]: purged_snaps scrub ok 2024-05-09T03:46:01.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:46:01.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[27629]: osd.2 [v2:172.21.15.78:6818/3785529399,v1:172.21.15.78:6819/3785529399] boot 2024-05-09T03:46:01.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[27629]: osdmap e18: 3 total, 3 up, 3 in 2024-05-09T03:46:01.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:46:01.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:00 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-09T03:46:01.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:01 smithi078 ceph-mon[27629]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:46:01.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:01 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-09T03:46:01.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:01 smithi078 ceph-mon[27629]: osdmap e19: 3 total, 3 up, 3 in 2024-05-09T03:46:01.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:01 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-09T03:46:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:01 smithi078 ceph-mon[22905]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:46:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:01 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-09T03:46:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:01 smithi078 ceph-mon[22905]: osdmap e19: 3 total, 3 up, 3 in 2024-05-09T03:46:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:01 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-09T03:46:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:01 smithi132 ceph-mon[23807]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-09T03:46:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:01 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-09T03:46:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:01 smithi132 ceph-mon[23807]: osdmap e19: 3 total, 3 up, 3 in 2024-05-09T03:46:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:01 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-09T03:46:02.480 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 03:46:02 smithi078 sudo[37612]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-09T03:46:02.480 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 03:46:02 smithi078 sudo[37612]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-09T03:46:02.480 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 03:46:02 smithi078 sudo[37612]: pam_unix(sudo:session): session closed for user root 2024-05-09T03:46:02.734 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 03:46:02 smithi078 sudo[37635]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-09T03:46:02.734 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 03:46:02 smithi078 sudo[37635]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-09T03:46:02.734 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 03:46:02 smithi078 sudo[37635]: pam_unix(sudo:session): session closed for user root 2024-05-09T03:46:02.989 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:02 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-09T03:46:02.989 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:02 smithi078 ceph-mon[27629]: osdmap e20: 3 total, 3 up, 3 in 2024-05-09T03:46:02.989 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:02 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:02.989 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 03:46:02 smithi078 sudo[37650]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-09T03:46:02.990 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 03:46:02 smithi078 sudo[37650]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-09T03:46:02.990 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 03:46:02 smithi078 sudo[37650]: pam_unix(sudo:session): session closed for user root 2024-05-09T03:46:02.990 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:02 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-09T03:46:02.990 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:02 smithi078 ceph-mon[22905]: osdmap e20: 3 total, 3 up, 3 in 2024-05-09T03:46:02.990 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:02 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:03.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:02 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-09T03:46:03.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:02 smithi132 ceph-mon[23807]: osdmap e20: 3 total, 3 up, 3 in 2024-05-09T03:46:03.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:02 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:03.262 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 03:46:03 smithi078 sudo[37700]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-09T03:46:03.262 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 03:46:03 smithi078 sudo[37700]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-09T03:46:03.262 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 03:46:03 smithi078 sudo[37700]: pam_unix(sudo:session): session closed for user root 2024-05-09T03:46:03.630 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 03:46:03 smithi078 sudo[37753]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-09T03:46:03.630 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 03:46:03 smithi078 sudo[37753]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-09T03:46:03.630 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 03:46:03 smithi078 sudo[37753]: pam_unix(sudo:session): session closed for user root 2024-05-09T03:46:03.930 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:03 smithi078 ceph-mon[27629]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:03.931 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:03 smithi078 ceph-mon[27629]: mgrmap e15: y(active, since 2m), standbys: x 2024-05-09T03:46:03.931 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:03 smithi078 ceph-mon[27629]: osdmap e21: 3 total, 3 up, 3 in 2024-05-09T03:46:03.931 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:03 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:46:03.931 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:03 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:46:03.931 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:03 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:03.931 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:03 smithi078 ceph-mon[22905]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:03.931 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:03 smithi078 ceph-mon[22905]: mgrmap e15: y(active, since 2m), standbys: x 2024-05-09T03:46:03.931 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:03 smithi078 ceph-mon[22905]: osdmap e21: 3 total, 3 up, 3 in 2024-05-09T03:46:03.932 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:03 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:46:03.932 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:03 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:46:03.932 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:03 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:03.932 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 03:46:03 smithi078 sudo[37770]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-09T03:46:03.932 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 03:46:03 smithi078 sudo[37770]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-09T03:46:03.932 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 03:46:03 smithi078 sudo[37770]: pam_unix(sudo:session): session closed for user root 2024-05-09T03:46:04.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:03 smithi132 ceph-mon[23807]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:04.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:03 smithi132 ceph-mon[23807]: mgrmap e15: y(active, since 2m), standbys: x 2024-05-09T03:46:04.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:03 smithi132 ceph-mon[23807]: osdmap e21: 3 total, 3 up, 3 in 2024-05-09T03:46:04.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:03 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:46:04.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:03 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:46:04.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:03 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:04.222 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:03 smithi078 sudo[37785]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-09T03:46:04.222 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:03 smithi078 sudo[37785]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-09T03:46:04.222 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:04 smithi078 sudo[37785]: pam_unix(sudo:session): session closed for user root 2024-05-09T03:46:04.506 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:04 smithi078 sudo[37810]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-09T03:46:04.506 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:04 smithi078 sudo[37810]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-09T03:46:04.506 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:04 smithi078 sudo[37810]: pam_unix(sudo:session): session closed for user root 2024-05-09T03:46:04.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:04 smithi078 sudo[37825]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-09T03:46:04.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:04 smithi078 sudo[37825]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-09T03:46:04.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:04 smithi078 sudo[37825]: pam_unix(sudo:session): session closed for user root 2024-05-09T03:46:05.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:04 smithi078 sudo[37832]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-09T03:46:05.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:04 smithi078 sudo[37832]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-09T03:46:05.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:04 smithi078 sudo[37832]: pam_unix(sudo:session): session closed for user root 2024-05-09T03:46:05.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 sudo[25327]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-09T03:46:05.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 sudo[25327]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-09T03:46:05.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 sudo[25327]: pam_unix(sudo:session): session closed for user root 2024-05-09T03:46:05.356 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:46:05.356 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-09T03:46:05.356 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: Detected new or changed devices on smithi078 2024-05-09T03:46:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-09T03:46:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:46:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:46:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:46:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:46:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:46:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:46:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[22905]: Adjusting osd_memory_target on smithi078 to 5354M 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: Detected new or changed devices on smithi078 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:05.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:05.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:05.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:05 smithi078 ceph-mon[27629]: Adjusting osd_memory_target on smithi078 to 5354M 2024-05-09T03:46:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 sudo[25330]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-09T03:46:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 sudo[25330]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-09T03:46:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 sudo[25330]: pam_unix(sudo:session): session closed for user root 2024-05-09T03:46:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:46:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-09T03:46:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: Detected new or changed devices on smithi078 2024-05-09T03:46:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-09T03:46:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:46:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:46:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:46:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:46:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:46:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:46:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:05 smithi132 ceph-mon[23807]: Adjusting osd_memory_target on smithi078 to 5354M 2024-05-09T03:46:06.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-09T03:46:06.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: pgmap v77: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:06.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-09T03:46:06.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:06.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-09T03:46:06.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:46:06.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:46:06.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:46:06.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-09T03:46:06.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:06.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:06.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:06.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:06.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:06.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:06.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2068135532' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97f610ec-76cf-4cb4-8a9a-f111e8b437c0"}]: dispatch 2024-05-09T03:46:06.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2068135532' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "97f610ec-76cf-4cb4-8a9a-f111e8b437c0"}]': finished 2024-05-09T03:46:06.757 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: osdmap e22: 4 total, 3 up, 4 in 2024-05-09T03:46:06.757 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:06 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:06.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-09T03:46:06.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: pgmap v77: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:06.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-09T03:46:06.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:06.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-09T03:46:06.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:46:06.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:46:06.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:46:06.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-09T03:46:06.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:06.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:06.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:06.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:06.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:06.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:06.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2068135532' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97f610ec-76cf-4cb4-8a9a-f111e8b437c0"}]: dispatch 2024-05-09T03:46:06.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2068135532' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "97f610ec-76cf-4cb4-8a9a-f111e8b437c0"}]': finished 2024-05-09T03:46:06.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: osdmap e22: 4 total, 3 up, 4 in 2024-05-09T03:46:06.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:06.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-09T03:46:06.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: pgmap v77: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:06.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-09T03:46:06.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:06.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-09T03:46:06.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:46:06.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:46:06.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:46:06.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-09T03:46:06.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:06.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:06.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:06.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:06.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:06.767 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:06.767 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2068135532' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97f610ec-76cf-4cb4-8a9a-f111e8b437c0"}]: dispatch 2024-05-09T03:46:06.767 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2068135532' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "97f610ec-76cf-4cb4-8a9a-f111e8b437c0"}]': finished 2024-05-09T03:46:06.767 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: osdmap e22: 4 total, 3 up, 4 in 2024-05-09T03:46:06.767 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:06 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:07.663 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:07 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1785878480' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:46:07.663 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:07 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1785878480' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:46:07.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:07 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1785878480' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:46:08.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:08 smithi078 ceph-mon[22905]: pgmap v79: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:08.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:08 smithi078 ceph-mon[27629]: pgmap v79: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:08 smithi132 ceph-mon[23807]: pgmap v79: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:10 smithi078 ceph-mon[22905]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2024-05-09T03:46:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:10 smithi078 ceph-mon[27629]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2024-05-09T03:46:10.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:10 smithi132 ceph-mon[23807]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2024-05-09T03:46:12.411 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:12 smithi078 ceph-mon[22905]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2024-05-09T03:46:12.411 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:12 smithi078 ceph-mon[27629]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2024-05-09T03:46:12.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:12 smithi132 ceph-mon[23807]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2024-05-09T03:46:13.403 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:13 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-09T03:46:13.403 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:13 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:13.404 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:13 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-09T03:46:13.404 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:13 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:13.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:13 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-09T03:46:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:13 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:14.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:14 smithi078 ceph-mon[27629]: Deploying daemon osd.3 on smithi078 2024-05-09T03:46:14.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:14 smithi078 ceph-mon[27629]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.8 KiB/s rd, 135 KiB/s wr, 9 op/s 2024-05-09T03:46:14.653 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:14 smithi078 ceph-mon[22905]: Deploying daemon osd.3 on smithi078 2024-05-09T03:46:14.653 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:14 smithi078 ceph-mon[22905]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.8 KiB/s rd, 135 KiB/s wr, 9 op/s 2024-05-09T03:46:14.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:14 smithi132 ceph-mon[23807]: Deploying daemon osd.3 on smithi078 2024-05-09T03:46:14.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:14 smithi132 ceph-mon[23807]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.8 KiB/s rd, 135 KiB/s wr, 9 op/s 2024-05-09T03:46:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:16 smithi078 ceph-mon[22905]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-05-09T03:46:16.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:16 smithi078 ceph-mon[27629]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-05-09T03:46:16.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:16 smithi132 ceph-mon[23807]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-05-09T03:46:18.538 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:18 smithi078 ceph-mon[27629]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 126 KiB/s wr, 8 op/s 2024-05-09T03:46:18.538 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:18 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:18.538 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:18 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:18.538 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:18 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:18.538 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:18 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:18.538 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:18 smithi078 ceph-mon[22905]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 126 KiB/s wr, 8 op/s 2024-05-09T03:46:18.539 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:18 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:18.539 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:18 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:18.539 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:18 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:18.539 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:18 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:18.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:18 smithi132 ceph-mon[23807]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 126 KiB/s wr, 8 op/s 2024-05-09T03:46:18.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:18 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:18.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:18 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:18.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:18 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:18.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:18 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:20.297 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 3 on host 'smithi078' 2024-05-09T03:46:20.653 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:20 smithi078 ceph-mon[22905]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-05-09T03:46:20.653 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:20 smithi078 ceph-mon[27629]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-05-09T03:46:20.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:20 smithi132 ceph-mon[23807]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-05-09T03:46:20.969 DEBUG:teuthology.orchestra.run.smithi078:osd.3> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.3.service 2024-05-09T03:46:20.971 INFO:tasks.cephadm:Deploying osd.4 on smithi132 with /dev/vg_nvme/lv_4... 2024-05-09T03:46:20.972 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-05-09T03:46:21.468 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:21 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:21.468 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:21 smithi078 ceph-mon[27629]: from='osd.3 [v2:172.21.15.78:6826/3522076329,v1:172.21.15.78:6827/3522076329]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-09T03:46:21.468 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:21 smithi078 ceph-mon[27629]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-09T03:46:21.468 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:21 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:21.468 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:21 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:21.468 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:21 smithi078 ceph-mon[22905]: from='osd.3 [v2:172.21.15.78:6826/3522076329,v1:172.21.15.78:6827/3522076329]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-09T03:46:21.469 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:21 smithi078 ceph-mon[22905]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-09T03:46:21.469 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:21 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:21.591 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:21 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:21.591 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:21 smithi132 ceph-mon[23807]: from='osd.3 [v2:172.21.15.78:6826/3522076329,v1:172.21.15.78:6827/3522076329]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-09T03:46:21.591 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:21 smithi132 ceph-mon[23807]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-09T03:46:21.591 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:21 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:22.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:22 smithi078 ceph-mon[22905]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:22.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:22 smithi078 ceph-mon[22905]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-09T03:46:22.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:22 smithi078 ceph-mon[22905]: osdmap e23: 4 total, 3 up, 4 in 2024-05-09T03:46:22.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:22 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:22.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:22 smithi078 ceph-mon[22905]: from='osd.3 [v2:172.21.15.78:6826/3522076329,v1:172.21.15.78:6827/3522076329]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:46:22.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:22 smithi078 ceph-mon[22905]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:46:22.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:22 smithi078 ceph-mon[27629]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:22.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:22 smithi078 ceph-mon[27629]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-09T03:46:22.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:22 smithi078 ceph-mon[27629]: osdmap e23: 4 total, 3 up, 4 in 2024-05-09T03:46:22.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:22 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:22.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:22 smithi078 ceph-mon[27629]: from='osd.3 [v2:172.21.15.78:6826/3522076329,v1:172.21.15.78:6827/3522076329]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:46:22.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:22 smithi078 ceph-mon[27629]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:46:22.515 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 03:46:22 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3[39353]: 2024-05-09T03:46:22.310+0000 7f2c55dee700 -1 osd.3 0 waiting for initial osdmap 2024-05-09T03:46:22.515 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 03:46:22 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3[39353]: 2024-05-09T03:46:22.314+0000 7f2c4ef82700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T03:46:22.572 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:22 smithi132 ceph-mon[23807]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:22.573 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:22 smithi132 ceph-mon[23807]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-09T03:46:22.573 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:22 smithi132 ceph-mon[23807]: osdmap e23: 4 total, 3 up, 4 in 2024-05-09T03:46:22.573 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:22 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:22.573 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:22 smithi132 ceph-mon[23807]: from='osd.3 [v2:172.21.15.78:6826/3522076329,v1:172.21.15.78:6827/3522076329]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:46:22.573 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:22 smithi132 ceph-mon[23807]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T03:46:23.702 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:23 smithi078 ceph-mon[27629]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-09T03:46:23.702 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:23 smithi078 ceph-mon[27629]: osdmap e24: 4 total, 3 up, 4 in 2024-05-09T03:46:23.702 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:23 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:23.703 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:23 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:23.703 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:23 smithi078 ceph-mon[22905]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-09T03:46:23.703 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:23 smithi078 ceph-mon[22905]: osdmap e24: 4 total, 3 up, 4 in 2024-05-09T03:46:23.703 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:23 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:23.703 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:23 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:23.703 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:23 smithi132 ceph-mon[23807]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-09T03:46:23.704 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:23 smithi132 ceph-mon[23807]: osdmap e24: 4 total, 3 up, 4 in 2024-05-09T03:46:23.704 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:23 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:23.704 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:23 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:23.707 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:46:23.725 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi132:vg_nvme/lv_4 2024-05-09T03:46:24.616 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:24 smithi132 ceph-mon[23807]: purged_snaps scrub starts 2024-05-09T03:46:24.616 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:24 smithi132 ceph-mon[23807]: purged_snaps scrub ok 2024-05-09T03:46:24.616 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:24 smithi132 ceph-mon[23807]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:24.617 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:24 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:24.617 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:24 smithi132 ceph-mon[23807]: osd.3 [v2:172.21.15.78:6826/3522076329,v1:172.21.15.78:6827/3522076329] boot 2024-05-09T03:46:24.617 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:24 smithi132 ceph-mon[23807]: osdmap e25: 4 total, 4 up, 4 in 2024-05-09T03:46:24.617 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:24 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[22905]: purged_snaps scrub starts 2024-05-09T03:46:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[22905]: purged_snaps scrub ok 2024-05-09T03:46:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[22905]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[22905]: osd.3 [v2:172.21.15.78:6826/3522076329,v1:172.21.15.78:6827/3522076329] boot 2024-05-09T03:46:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[22905]: osdmap e25: 4 total, 4 up, 4 in 2024-05-09T03:46:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:24.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[27629]: purged_snaps scrub starts 2024-05-09T03:46:24.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[27629]: purged_snaps scrub ok 2024-05-09T03:46:24.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[27629]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-09T03:46:24.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:24.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[27629]: osd.3 [v2:172.21.15.78:6826/3522076329,v1:172.21.15.78:6827/3522076329] boot 2024-05-09T03:46:24.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[27629]: osdmap e25: 4 total, 4 up, 4 in 2024-05-09T03:46:24.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:24 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:46:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: osdmap e26: 4 total, 4 up, 4 in 2024-05-09T03:46:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: Detected new or changed devices on smithi078 2024-05-09T03:46:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:25.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:25.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:25.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:25.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:25.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:25 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: osdmap e26: 4 total, 4 up, 4 in 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: Detected new or changed devices on smithi078 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:25.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: osdmap e26: 4 total, 4 up, 4 in 2024-05-09T03:46:25.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: Detected new or changed devices on smithi078 2024-05-09T03:46:25.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:25.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:25.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:25.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:25.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:25.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:25.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:25.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:25.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:25.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:25.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:25 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:26.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:26 smithi132 ceph-mon[23807]: Adjusting osd_memory_target on smithi078 to 4016M 2024-05-09T03:46:26.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:26 smithi132 ceph-mon[23807]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:26.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:26 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:46:26.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:26 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:46:26.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:26 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:26 smithi078 ceph-mon[27629]: Adjusting osd_memory_target on smithi078 to 4016M 2024-05-09T03:46:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:26 smithi078 ceph-mon[27629]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:26 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:46:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:26 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:46:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:26 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:26 smithi078 ceph-mon[22905]: Adjusting osd_memory_target on smithi078 to 4016M 2024-05-09T03:46:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:26 smithi078 ceph-mon[22905]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:26 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:46:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:26 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:46:26.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:26 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:27.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:27 smithi132 ceph-mon[23807]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi132:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:46:27.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:27 smithi078 ceph-mon[22905]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi132:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:46:27.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:27 smithi078 ceph-mon[27629]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi132:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:46:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:28 smithi132 ceph-mon[23807]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:28 smithi132 ceph-mon[23807]: from='client.? 172.21.15.132:0/2845784351' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd1bd685-528f-41bd-97fc-5aabda193ce2"}]: dispatch 2024-05-09T03:46:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:28 smithi132 ceph-mon[23807]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd1bd685-528f-41bd-97fc-5aabda193ce2"}]: dispatch 2024-05-09T03:46:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:28 smithi132 ceph-mon[23807]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bd1bd685-528f-41bd-97fc-5aabda193ce2"}]': finished 2024-05-09T03:46:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:28 smithi132 ceph-mon[23807]: osdmap e27: 5 total, 4 up, 5 in 2024-05-09T03:46:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:28 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:28 smithi078 ceph-mon[22905]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:28 smithi078 ceph-mon[22905]: from='client.? 172.21.15.132:0/2845784351' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd1bd685-528f-41bd-97fc-5aabda193ce2"}]: dispatch 2024-05-09T03:46:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:28 smithi078 ceph-mon[22905]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd1bd685-528f-41bd-97fc-5aabda193ce2"}]: dispatch 2024-05-09T03:46:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:28 smithi078 ceph-mon[22905]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bd1bd685-528f-41bd-97fc-5aabda193ce2"}]': finished 2024-05-09T03:46:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:28 smithi078 ceph-mon[22905]: osdmap e27: 5 total, 4 up, 5 in 2024-05-09T03:46:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:28 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:28 smithi078 ceph-mon[27629]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:28.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:28 smithi078 ceph-mon[27629]: from='client.? 172.21.15.132:0/2845784351' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd1bd685-528f-41bd-97fc-5aabda193ce2"}]: dispatch 2024-05-09T03:46:28.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:28 smithi078 ceph-mon[27629]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd1bd685-528f-41bd-97fc-5aabda193ce2"}]: dispatch 2024-05-09T03:46:28.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:28 smithi078 ceph-mon[27629]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bd1bd685-528f-41bd-97fc-5aabda193ce2"}]': finished 2024-05-09T03:46:28.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:28 smithi078 ceph-mon[27629]: osdmap e27: 5 total, 4 up, 5 in 2024-05-09T03:46:28.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:28 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:29.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:29 smithi132 ceph-mon[23807]: from='client.? 172.21.15.132:0/3866097297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:46:29.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:29 smithi078 ceph-mon[22905]: from='client.? 172.21.15.132:0/3866097297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:46:29.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:29 smithi078 ceph-mon[27629]: from='client.? 172.21.15.132:0/3866097297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:46:30.599 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:30 smithi132 ceph-mon[23807]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:30.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:30 smithi078 ceph-mon[22905]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:30.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:30 smithi078 ceph-mon[27629]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:32.709 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:32 smithi132 ceph-mon[23807]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:32.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:32 smithi078 ceph-mon[22905]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:32.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:32 smithi078 ceph-mon[27629]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:33 smithi132 ceph-mon[23807]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:34.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:33 smithi078 ceph-mon[22905]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:34.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:33 smithi078 ceph-mon[27629]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:35.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:34 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-09T03:46:35.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:34 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:35.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:34 smithi132 ceph-mon[23807]: Deploying daemon osd.4 on smithi132 2024-05-09T03:46:35.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:34 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-09T03:46:35.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:34 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:35.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:34 smithi078 ceph-mon[22905]: Deploying daemon osd.4 on smithi132 2024-05-09T03:46:35.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:34 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-09T03:46:35.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:34 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:35.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:34 smithi078 ceph-mon[27629]: Deploying daemon osd.4 on smithi132 2024-05-09T03:46:36.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:35 smithi132 ceph-mon[23807]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:36.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:35 smithi078 ceph-mon[22905]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:36.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:35 smithi078 ceph-mon[27629]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:38.192 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:37 smithi132 ceph-mon[23807]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:38.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:37 smithi078 ceph-mon[22905]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:38.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:37 smithi078 ceph-mon[27629]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:40.316 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:40 smithi132 ceph-mon[23807]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:40.316 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:40.316 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:40.316 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:40.316 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:40 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:40.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:40 smithi078 ceph-mon[22905]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:40.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:40.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:40.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:40.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:40 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:40.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:40 smithi078 ceph-mon[27629]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:40.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:40.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:40.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:40.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:40 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:42.146 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:42 smithi132 ceph-mon[23807]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:42.178 INFO:teuthology.orchestra.run.smithi132.stdout:Created osd(s) 4 on host 'smithi132' 2024-05-09T03:46:42.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:42 smithi078 ceph-mon[22905]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:42.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:42 smithi078 ceph-mon[27629]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:42.866 DEBUG:teuthology.orchestra.run.smithi132:osd.4> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.4.service 2024-05-09T03:46:42.869 INFO:tasks.cephadm:Deploying osd.5 on smithi132 with /dev/vg_nvme/lv_3... 2024-05-09T03:46:42.869 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-05-09T03:46:43.222 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:43 smithi132 ceph-mon[23807]: from='osd.4 [v2:172.21.15.132:6800/268288079,v1:172.21.15.132:6801/268288079]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-09T03:46:43.222 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:43 smithi132 ceph-mon[23807]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-09T03:46:43.222 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:43 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:43.222 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:43 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:43 smithi078 ceph-mon[22905]: from='osd.4 [v2:172.21.15.132:6800/268288079,v1:172.21.15.132:6801/268288079]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-09T03:46:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:43 smithi078 ceph-mon[22905]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-09T03:46:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:43 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:43 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:43 smithi078 ceph-mon[27629]: from='osd.4 [v2:172.21.15.132:6800/268288079,v1:172.21.15.132:6801/268288079]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-09T03:46:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:43 smithi078 ceph-mon[27629]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-09T03:46:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:43 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:43 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:44.229 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:44 smithi132 ceph-mon[23807]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:44.230 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:44 smithi132 ceph-mon[23807]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-09T03:46:44.230 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:44 smithi132 ceph-mon[23807]: osdmap e28: 5 total, 4 up, 5 in 2024-05-09T03:46:44.230 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:44 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:44.230 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:44 smithi132 ceph-mon[23807]: from='osd.4 [v2:172.21.15.132:6800/268288079,v1:172.21.15.132:6801/268288079]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:46:44.230 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:44 smithi132 ceph-mon[23807]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:46:44.230 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 03:46:44 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[27170]: 2024-05-09T03:46:44.070+0000 7fd4833e1700 -1 osd.4 0 waiting for initial osdmap 2024-05-09T03:46:44.230 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 03:46:44 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[27170]: 2024-05-09T03:46:44.074+0000 7fd47c575700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T03:46:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:44 smithi078 ceph-mon[22905]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:44 smithi078 ceph-mon[22905]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-09T03:46:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:44 smithi078 ceph-mon[22905]: osdmap e28: 5 total, 4 up, 5 in 2024-05-09T03:46:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:44 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:44 smithi078 ceph-mon[22905]: from='osd.4 [v2:172.21.15.132:6800/268288079,v1:172.21.15.132:6801/268288079]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:46:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:44 smithi078 ceph-mon[22905]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:46:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:44 smithi078 ceph-mon[27629]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:44 smithi078 ceph-mon[27629]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-09T03:46:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:44 smithi078 ceph-mon[27629]: osdmap e28: 5 total, 4 up, 5 in 2024-05-09T03:46:44.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:44 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:44.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:44 smithi078 ceph-mon[27629]: from='osd.4 [v2:172.21.15.132:6800/268288079,v1:172.21.15.132:6801/268288079]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:46:44.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:44 smithi078 ceph-mon[27629]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:46:45.097 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:45 smithi132 ceph-mon[23807]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]': finished 2024-05-09T03:46:45.098 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:45 smithi132 ceph-mon[23807]: osdmap e29: 5 total, 4 up, 5 in 2024-05-09T03:46:45.098 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:45 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:45.098 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:45 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:45 smithi078 ceph-mon[27629]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]': finished 2024-05-09T03:46:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:45 smithi078 ceph-mon[27629]: osdmap e29: 5 total, 4 up, 5 in 2024-05-09T03:46:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:45 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:45 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:45 smithi078 ceph-mon[22905]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]': finished 2024-05-09T03:46:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:45 smithi078 ceph-mon[22905]: osdmap e29: 5 total, 4 up, 5 in 2024-05-09T03:46:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:45 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:45 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:46.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:46 smithi132 ceph-mon[23807]: purged_snaps scrub starts 2024-05-09T03:46:46.397 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:46 smithi132 ceph-mon[23807]: purged_snaps scrub ok 2024-05-09T03:46:46.397 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:46 smithi132 ceph-mon[23807]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:46.397 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:46.397 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:46 smithi132 ceph-mon[23807]: osd.4 [v2:172.21.15.132:6800/268288079,v1:172.21.15.132:6801/268288079] boot 2024-05-09T03:46:46.397 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:46 smithi132 ceph-mon[23807]: osdmap e30: 5 total, 5 up, 5 in 2024-05-09T03:46:46.397 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:46 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:46.402 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:46:46.425 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi132:vg_nvme/lv_3 2024-05-09T03:46:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[27629]: purged_snaps scrub starts 2024-05-09T03:46:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[27629]: purged_snaps scrub ok 2024-05-09T03:46:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[27629]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[27629]: osd.4 [v2:172.21.15.132:6800/268288079,v1:172.21.15.132:6801/268288079] boot 2024-05-09T03:46:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[27629]: osdmap e30: 5 total, 5 up, 5 in 2024-05-09T03:46:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:46.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[22905]: purged_snaps scrub starts 2024-05-09T03:46:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[22905]: purged_snaps scrub ok 2024-05-09T03:46:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[22905]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-09T03:46:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[22905]: osd.4 [v2:172.21.15.132:6800/268288079,v1:172.21.15.132:6801/268288079] boot 2024-05-09T03:46:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[22905]: osdmap e30: 5 total, 5 up, 5 in 2024-05-09T03:46:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:46 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:46:47.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:47 smithi132 ceph-mon[23807]: osdmap e31: 5 total, 5 up, 5 in 2024-05-09T03:46:47.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:47 smithi078 ceph-mon[22905]: osdmap e31: 5 total, 5 up, 5 in 2024-05-09T03:46:47.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:47 smithi078 ceph-mon[27629]: osdmap e31: 5 total, 5 up, 5 in 2024-05-09T03:46:48.251 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:48 smithi132 ceph-mon[23807]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:46:48.251 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:48 smithi132 ceph-mon[23807]: osdmap e32: 5 total, 5 up, 5 in 2024-05-09T03:46:48.251 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:48 smithi132 ceph-mon[23807]: osdmap e33: 5 total, 5 up, 5 in 2024-05-09T03:46:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:48 smithi078 ceph-mon[22905]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:46:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:48 smithi078 ceph-mon[22905]: osdmap e32: 5 total, 5 up, 5 in 2024-05-09T03:46:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:48 smithi078 ceph-mon[22905]: osdmap e33: 5 total, 5 up, 5 in 2024-05-09T03:46:48.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:48 smithi078 ceph-mon[27629]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:46:48.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:48 smithi078 ceph-mon[27629]: osdmap e32: 5 total, 5 up, 5 in 2024-05-09T03:46:48.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:48 smithi078 ceph-mon[27629]: osdmap e33: 5 total, 5 up, 5 in 2024-05-09T03:46:49.352 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:49 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:46:49.352 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:49 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:46:49.352 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:49 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:49.352 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:49 smithi132 ceph-mon[23807]: osdmap e34: 5 total, 5 up, 5 in 2024-05-09T03:46:49.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:49 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:46:49.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:49 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:46:49.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:49 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:49.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:49 smithi078 ceph-mon[27629]: osdmap e34: 5 total, 5 up, 5 in 2024-05-09T03:46:49.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:49 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:46:49.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:49 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:46:49.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:49 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:49.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:49 smithi078 ceph-mon[22905]: osdmap e34: 5 total, 5 up, 5 in 2024-05-09T03:46:50.359 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:50 smithi132 ceph-mon[23807]: pgmap v111: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-09T03:46:50.360 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:50 smithi132 ceph-mon[23807]: from='client.24218 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi132:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:46:50.360 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:50 smithi132 ceph-mon[23807]: Detected new or changed devices on smithi132 2024-05-09T03:46:50.360 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:50 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:50.360 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:50 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:50.360 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:50 smithi132 ceph-mon[23807]: Adjusting osd_memory_target on smithi132 to 17088M 2024-05-09T03:46:50.360 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:50 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:50.360 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:50 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:50.360 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:50 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:50.360 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:50 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:50.360 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:50 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:50.360 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:50 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[22905]: pgmap v111: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-09T03:46:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[22905]: from='client.24218 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi132:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:46:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[22905]: Detected new or changed devices on smithi132 2024-05-09T03:46:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[22905]: Adjusting osd_memory_target on smithi132 to 17088M 2024-05-09T03:46:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:50.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[27629]: pgmap v111: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-09T03:46:50.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[27629]: from='client.24218 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi132:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:46:50.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[27629]: Detected new or changed devices on smithi132 2024-05-09T03:46:50.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:50.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:46:50.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[27629]: Adjusting osd_memory_target on smithi132 to 17088M 2024-05-09T03:46:50.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:50.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:50.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:46:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:46:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:50 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:46:52.428 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[27629]: pgmap v113: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 119 KiB/s, 0 objects/s recovering 2024-05-09T03:46:52.429 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[27629]: from='client.? 172.21.15.132:0/3413815100' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30240d46-0459-4fde-ac42-1c4a82b483ee"}]: dispatch 2024-05-09T03:46:52.429 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[27629]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30240d46-0459-4fde-ac42-1c4a82b483ee"}]: dispatch 2024-05-09T03:46:52.429 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[27629]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "30240d46-0459-4fde-ac42-1c4a82b483ee"}]': finished 2024-05-09T03:46:52.429 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[27629]: osdmap e35: 6 total, 5 up, 6 in 2024-05-09T03:46:52.429 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:46:52.429 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[27629]: from='client.? 172.21.15.132:0/2414378514' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:46:52.429 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[22905]: pgmap v113: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 119 KiB/s, 0 objects/s recovering 2024-05-09T03:46:52.429 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[22905]: from='client.? 172.21.15.132:0/3413815100' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30240d46-0459-4fde-ac42-1c4a82b483ee"}]: dispatch 2024-05-09T03:46:52.429 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[22905]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30240d46-0459-4fde-ac42-1c4a82b483ee"}]: dispatch 2024-05-09T03:46:52.430 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[22905]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "30240d46-0459-4fde-ac42-1c4a82b483ee"}]': finished 2024-05-09T03:46:52.430 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[22905]: osdmap e35: 6 total, 5 up, 6 in 2024-05-09T03:46:52.430 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:46:52.430 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:52 smithi078 ceph-mon[22905]: from='client.? 172.21.15.132:0/2414378514' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:46:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:52 smithi132 ceph-mon[23807]: pgmap v113: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 119 KiB/s, 0 objects/s recovering 2024-05-09T03:46:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:52 smithi132 ceph-mon[23807]: from='client.? 172.21.15.132:0/3413815100' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30240d46-0459-4fde-ac42-1c4a82b483ee"}]: dispatch 2024-05-09T03:46:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:52 smithi132 ceph-mon[23807]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30240d46-0459-4fde-ac42-1c4a82b483ee"}]: dispatch 2024-05-09T03:46:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:52 smithi132 ceph-mon[23807]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "30240d46-0459-4fde-ac42-1c4a82b483ee"}]': finished 2024-05-09T03:46:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:52 smithi132 ceph-mon[23807]: osdmap e35: 6 total, 5 up, 6 in 2024-05-09T03:46:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:46:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:52 smithi132 ceph-mon[23807]: from='client.? 172.21.15.132:0/2414378514' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:46:52.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:46:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:46:52.426+0000 7f24d81e1700 -1 calc_pg_upmaps abort due to max <= 0 2024-05-09T03:46:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:53 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:46:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:53 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:46:53.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:46:53.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:53 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:46:53.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:53 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:46:53.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:53 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:46:54.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:54 smithi132 ceph-mon[23807]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-05-09T03:46:54.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:54 smithi078 ceph-mon[22905]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-05-09T03:46:54.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:54 smithi078 ceph-mon[27629]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-05-09T03:46:56.178 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:56 smithi132 ceph-mon[23807]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:46:56.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:56 smithi078 ceph-mon[27629]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:46:56.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:56 smithi078 ceph-mon[22905]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:46:58.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:58 smithi132 ceph-mon[23807]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:46:58.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:58 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-09T03:46:58.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:58 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:58.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:58 smithi078 ceph-mon[22905]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:46:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:58 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-09T03:46:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:58 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:58 smithi078 ceph-mon[27629]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:46:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:58 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-09T03:46:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:58 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:46:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:46:59 smithi132 ceph-mon[23807]: Deploying daemon osd.5 on smithi132 2024-05-09T03:46:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:46:59 smithi078 ceph-mon[22905]: Deploying daemon osd.5 on smithi132 2024-05-09T03:46:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:46:59 smithi078 ceph-mon[27629]: Deploying daemon osd.5 on smithi132 2024-05-09T03:47:00.495 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:00 smithi132 ceph-mon[23807]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:00.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:00 smithi078 ceph-mon[22905]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:00.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:00 smithi078 ceph-mon[27629]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:02.300 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:02 smithi132 ceph-mon[23807]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:02.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:02 smithi078 ceph-mon[22905]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:02 smithi078 ceph-mon[27629]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:03.721 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:03 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:03.721 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:03 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:03.721 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:03 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:03.722 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:03 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:03.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:03 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:03.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:03 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:03.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:03 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:03.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:03 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:03.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:03 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:03.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:03 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:03.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:03 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:03.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:03 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:04.655 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:04 smithi132 ceph-mon[23807]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:04.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:04 smithi078 ceph-mon[22905]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:04.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:04 smithi078 ceph-mon[27629]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:05.682 INFO:teuthology.orchestra.run.smithi132.stdout:Created osd(s) 5 on host 'smithi132' 2024-05-09T03:47:05.996 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:05 smithi132 ceph-mon[23807]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:05.996 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:05.996 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:05.996 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:05 smithi132 ceph-mon[23807]: from='osd.5 [v2:172.21.15.132:6808/968710714,v1:172.21.15.132:6809/968710714]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-09T03:47:05.996 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:05 smithi132 ceph-mon[23807]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-09T03:47:05.996 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:05.996 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:05.996 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:05.996 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:05 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:06.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[22905]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:06.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:06.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:06.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[22905]: from='osd.5 [v2:172.21.15.132:6808/968710714,v1:172.21.15.132:6809/968710714]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-09T03:47:06.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[22905]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-09T03:47:06.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:06.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:06.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:06.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:06.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[27629]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:06.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:06.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:06.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[27629]: from='osd.5 [v2:172.21.15.132:6808/968710714,v1:172.21.15.132:6809/968710714]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-09T03:47:06.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[27629]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-09T03:47:06.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:06.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:06.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:06.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:05 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:06.319 DEBUG:teuthology.orchestra.run.smithi132:osd.5> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.5.service 2024-05-09T03:47:06.321 INFO:tasks.cephadm:Deploying osd.6 on smithi132 with /dev/vg_nvme/lv_2... 2024-05-09T03:47:06.321 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-05-09T03:47:07.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:07 smithi132 ceph-mon[23807]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-09T03:47:07.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:07 smithi132 ceph-mon[23807]: osdmap e36: 6 total, 5 up, 6 in 2024-05-09T03:47:07.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:07 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:47:07.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:07 smithi132 ceph-mon[23807]: from='osd.5 [v2:172.21.15.132:6808/968710714,v1:172.21.15.132:6809/968710714]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:07.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:07 smithi132 ceph-mon[23807]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:07.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:07 smithi132 ceph-mon[23807]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:07.505 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 03:47:07 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[30119]: 2024-05-09T03:47:07.364+0000 7f4ad5950700 -1 osd.5 0 waiting for initial osdmap 2024-05-09T03:47:07.505 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 03:47:07 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[30119]: 2024-05-09T03:47:07.369+0000 7f4ace2e3700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T03:47:07.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:07 smithi078 ceph-mon[22905]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-09T03:47:07.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:07 smithi078 ceph-mon[22905]: osdmap e36: 6 total, 5 up, 6 in 2024-05-09T03:47:07.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:07 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:47:07.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:07 smithi078 ceph-mon[22905]: from='osd.5 [v2:172.21.15.132:6808/968710714,v1:172.21.15.132:6809/968710714]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:07.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:07 smithi078 ceph-mon[22905]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:07.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:07 smithi078 ceph-mon[22905]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:07.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:07 smithi078 ceph-mon[27629]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-09T03:47:07.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:07 smithi078 ceph-mon[27629]: osdmap e36: 6 total, 5 up, 6 in 2024-05-09T03:47:07.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:07 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:47:07.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:07 smithi078 ceph-mon[27629]: from='osd.5 [v2:172.21.15.132:6808/968710714,v1:172.21.15.132:6809/968710714]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:07.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:07 smithi078 ceph-mon[27629]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:07.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:07 smithi078 ceph-mon[27629]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-09T03:47:08.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[22905]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]': finished 2024-05-09T03:47:08.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[22905]: osdmap e37: 6 total, 5 up, 6 in 2024-05-09T03:47:08.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:47:08.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:47:08.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[22905]: osd.5 [v2:172.21.15.132:6808/968710714,v1:172.21.15.132:6809/968710714] boot 2024-05-09T03:47:08.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[22905]: osdmap e38: 6 total, 6 up, 6 in 2024-05-09T03:47:08.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:47:08.665 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[27629]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]': finished 2024-05-09T03:47:08.665 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[27629]: osdmap e37: 6 total, 5 up, 6 in 2024-05-09T03:47:08.665 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:47:08.665 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:47:08.665 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[27629]: osd.5 [v2:172.21.15.132:6808/968710714,v1:172.21.15.132:6809/968710714] boot 2024-05-09T03:47:08.665 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[27629]: osdmap e38: 6 total, 6 up, 6 in 2024-05-09T03:47:08.665 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:08 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:47:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:08 smithi132 ceph-mon[23807]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]': finished 2024-05-09T03:47:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:08 smithi132 ceph-mon[23807]: osdmap e37: 6 total, 5 up, 6 in 2024-05-09T03:47:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:08 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:47:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:08 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:47:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:08 smithi132 ceph-mon[23807]: osd.5 [v2:172.21.15.132:6808/968710714,v1:172.21.15.132:6809/968710714] boot 2024-05-09T03:47:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:08 smithi132 ceph-mon[23807]: osdmap e38: 6 total, 6 up, 6 in 2024-05-09T03:47:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:08 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:47:09.623 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:09 smithi132 ceph-mon[23807]: purged_snaps scrub starts 2024-05-09T03:47:09.623 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:09 smithi132 ceph-mon[23807]: purged_snaps scrub ok 2024-05-09T03:47:09.623 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:09 smithi132 ceph-mon[23807]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:09.623 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:09 smithi132 ceph-mon[23807]: osdmap e39: 6 total, 6 up, 6 in 2024-05-09T03:47:09.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:09 smithi078 ceph-mon[27629]: purged_snaps scrub starts 2024-05-09T03:47:09.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:09 smithi078 ceph-mon[27629]: purged_snaps scrub ok 2024-05-09T03:47:09.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:09 smithi078 ceph-mon[27629]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:09.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:09 smithi078 ceph-mon[27629]: osdmap e39: 6 total, 6 up, 6 in 2024-05-09T03:47:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:09 smithi078 ceph-mon[22905]: purged_snaps scrub starts 2024-05-09T03:47:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:09 smithi078 ceph-mon[22905]: purged_snaps scrub ok 2024-05-09T03:47:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:09 smithi078 ceph-mon[22905]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:09 smithi078 ceph-mon[22905]: osdmap e39: 6 total, 6 up, 6 in 2024-05-09T03:47:10.629 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:10 smithi132 ceph-mon[23807]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-05-09T03:47:10.630 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:10 smithi132 ceph-mon[23807]: osdmap e40: 6 total, 6 up, 6 in 2024-05-09T03:47:10.630 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:10 smithi132 ceph-mon[23807]: Detected new or changed devices on smithi132 2024-05-09T03:47:10.630 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:10 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:10.630 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:10 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:10.630 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:10 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:10.630 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:10 smithi132 ceph-mon[23807]: Adjusting osd_memory_target on smithi132 to 8544M 2024-05-09T03:47:10.630 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:10 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:10.630 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:10 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[27629]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-05-09T03:47:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[27629]: osdmap e40: 6 total, 6 up, 6 in 2024-05-09T03:47:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[27629]: Detected new or changed devices on smithi132 2024-05-09T03:47:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[27629]: Adjusting osd_memory_target on smithi132 to 8544M 2024-05-09T03:47:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:10.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[22905]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-05-09T03:47:10.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[22905]: osdmap e40: 6 total, 6 up, 6 in 2024-05-09T03:47:10.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[22905]: Detected new or changed devices on smithi132 2024-05-09T03:47:10.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:10.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:10.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:10.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[22905]: Adjusting osd_memory_target on smithi132 to 8544M 2024-05-09T03:47:10.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:10.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:10 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:11.700 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:11 smithi132 ceph-mon[23807]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:11.705 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:47:11.726 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi132:vg_nvme/lv_2 2024-05-09T03:47:11.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:11 smithi078 ceph-mon[22905]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:11 smithi078 ceph-mon[27629]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:12.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:12 smithi132 ceph-mon[23807]: osdmap e41: 6 total, 6 up, 6 in 2024-05-09T03:47:12.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:12 smithi078 ceph-mon[22905]: osdmap e41: 6 total, 6 up, 6 in 2024-05-09T03:47:12.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:12 smithi078 ceph-mon[27629]: osdmap e41: 6 total, 6 up, 6 in 2024-05-09T03:47:13.689 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:13 smithi132 ceph-mon[23807]: osdmap e42: 6 total, 6 up, 6 in 2024-05-09T03:47:13.689 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:13 smithi132 ceph-mon[23807]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:13.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:13 smithi078 ceph-mon[22905]: osdmap e42: 6 total, 6 up, 6 in 2024-05-09T03:47:13.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:13 smithi078 ceph-mon[22905]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:13 smithi078 ceph-mon[27629]: osdmap e42: 6 total, 6 up, 6 in 2024-05-09T03:47:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:13 smithi078 ceph-mon[27629]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:14.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:14 smithi132 ceph-mon[23807]: from='client.24245 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi132:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:47:14.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:14 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:47:14.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:14 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:47:14.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:14 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:14.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:14 smithi078 ceph-mon[22905]: from='client.24245 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi132:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:47:14.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:14 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:47:14.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:14 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:47:14.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:14 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:14.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:14 smithi078 ceph-mon[27629]: from='client.24245 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi132:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:47:14.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:14 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:47:14.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:14 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:47:14.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:14 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:15.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:15 smithi132 ceph-mon[23807]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-09T03:47:15.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:15 smithi078 ceph-mon[22905]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-09T03:47:15.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:15 smithi078 ceph-mon[27629]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-09T03:47:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:16 smithi132 ceph-mon[23807]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-09T03:47:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:16 smithi132 ceph-mon[23807]: Cluster is now healthy 2024-05-09T03:47:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:16 smithi132 ceph-mon[23807]: from='client.? 172.21.15.132:0/852164019' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d6b0f898-9036-4b02-9cba-08e377b14063"}]: dispatch 2024-05-09T03:47:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:16 smithi132 ceph-mon[23807]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d6b0f898-9036-4b02-9cba-08e377b14063"}]: dispatch 2024-05-09T03:47:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:16 smithi132 ceph-mon[23807]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d6b0f898-9036-4b02-9cba-08e377b14063"}]': finished 2024-05-09T03:47:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:16 smithi132 ceph-mon[23807]: osdmap e43: 7 total, 6 up, 7 in 2024-05-09T03:47:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:16 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[22905]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-09T03:47:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[22905]: Cluster is now healthy 2024-05-09T03:47:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[22905]: from='client.? 172.21.15.132:0/852164019' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d6b0f898-9036-4b02-9cba-08e377b14063"}]: dispatch 2024-05-09T03:47:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[22905]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d6b0f898-9036-4b02-9cba-08e377b14063"}]: dispatch 2024-05-09T03:47:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[22905]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d6b0f898-9036-4b02-9cba-08e377b14063"}]': finished 2024-05-09T03:47:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[22905]: osdmap e43: 7 total, 6 up, 7 in 2024-05-09T03:47:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:16.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[27629]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-09T03:47:16.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[27629]: Cluster is now healthy 2024-05-09T03:47:16.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[27629]: from='client.? 172.21.15.132:0/852164019' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d6b0f898-9036-4b02-9cba-08e377b14063"}]: dispatch 2024-05-09T03:47:16.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[27629]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d6b0f898-9036-4b02-9cba-08e377b14063"}]: dispatch 2024-05-09T03:47:16.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[27629]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d6b0f898-9036-4b02-9cba-08e377b14063"}]': finished 2024-05-09T03:47:16.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[27629]: osdmap e43: 7 total, 6 up, 7 in 2024-05-09T03:47:16.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:16 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:17.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:17 smithi132 ceph-mon[23807]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-09T03:47:17.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:17 smithi132 ceph-mon[23807]: from='client.? 172.21.15.132:0/1139087790' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:47:17.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:17 smithi078 ceph-mon[22905]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-09T03:47:17.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:17 smithi078 ceph-mon[22905]: from='client.? 172.21.15.132:0/1139087790' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:47:17.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:17 smithi078 ceph-mon[27629]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-09T03:47:17.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:17 smithi078 ceph-mon[27629]: from='client.? 172.21.15.132:0/1139087790' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:47:20.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:19 smithi132 ceph-mon[23807]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-05-09T03:47:20.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:19 smithi078 ceph-mon[22905]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-05-09T03:47:20.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:19 smithi078 ceph-mon[27629]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-05-09T03:47:22.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:21 smithi132 ceph-mon[23807]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-05-09T03:47:22.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:21 smithi078 ceph-mon[22905]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-05-09T03:47:22.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:21 smithi078 ceph-mon[27629]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-05-09T03:47:23.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:22 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-09T03:47:23.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:22 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:23.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:22 smithi132 ceph-mon[23807]: Deploying daemon osd.6 on smithi132 2024-05-09T03:47:23.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:22 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-09T03:47:23.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:22 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:23.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:22 smithi078 ceph-mon[22905]: Deploying daemon osd.6 on smithi132 2024-05-09T03:47:23.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:22 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-09T03:47:23.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:22 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:23.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:22 smithi078 ceph-mon[27629]: Deploying daemon osd.6 on smithi132 2024-05-09T03:47:24.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:23 smithi132 ceph-mon[23807]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-09T03:47:24.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:23 smithi078 ceph-mon[27629]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-09T03:47:24.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:23 smithi078 ceph-mon[22905]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-09T03:47:26.187 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:25 smithi132 ceph-mon[23807]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:26.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:25 smithi078 ceph-mon[22905]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:25 smithi078 ceph-mon[27629]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:28.156 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:28 smithi132 ceph-mon[23807]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:28.156 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:28 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:28 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:28 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:28 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:28 smithi078 ceph-mon[22905]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:28 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:28 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:28.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:28 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:28.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:28 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:28 smithi078 ceph-mon[27629]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:28 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:28 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:28 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:28 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:30.502 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:30 smithi132 ceph-mon[23807]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:30.502 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:30 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:30.502 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:30 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:30.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:30 smithi078 ceph-mon[27629]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:30.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:30 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:30.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:30 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:30 smithi078 ceph-mon[22905]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:30 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:30 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:30.834 INFO:teuthology.orchestra.run.smithi132.stdout:Created osd(s) 6 on host 'smithi132' 2024-05-09T03:47:31.415 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:31 smithi132 ceph-mon[23807]: from='osd.6 [v2:172.21.15.132:6816/1013115018,v1:172.21.15.132:6817/1013115018]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-09T03:47:31.415 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:31 smithi132 ceph-mon[23807]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-09T03:47:31.416 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:31 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:31.416 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:31 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:31.416 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:31 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:31.416 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:31 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:31.438 DEBUG:teuthology.orchestra.run.smithi132:osd.6> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.6.service 2024-05-09T03:47:31.440 INFO:tasks.cephadm:Deploying osd.7 on smithi132 with /dev/vg_nvme/lv_1... 2024-05-09T03:47:31.441 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-05-09T03:47:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:31 smithi078 ceph-mon[22905]: from='osd.6 [v2:172.21.15.132:6816/1013115018,v1:172.21.15.132:6817/1013115018]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-09T03:47:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:31 smithi078 ceph-mon[22905]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-09T03:47:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:31 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:31 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:31 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:31 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:31 smithi078 ceph-mon[27629]: from='osd.6 [v2:172.21.15.132:6816/1013115018,v1:172.21.15.132:6817/1013115018]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-09T03:47:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:31 smithi078 ceph-mon[27629]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-09T03:47:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:31 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:31 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:31 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:31 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:32.424 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:32 smithi132 ceph-mon[23807]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:32.424 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:32 smithi132 ceph-mon[23807]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-09T03:47:32.424 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:32 smithi132 ceph-mon[23807]: osdmap e44: 7 total, 6 up, 7 in 2024-05-09T03:47:32.424 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:32 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:32.424 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:32 smithi132 ceph-mon[23807]: from='osd.6 [v2:172.21.15.132:6816/1013115018,v1:172.21.15.132:6817/1013115018]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:32.425 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:32 smithi132 ceph-mon[23807]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:32.425 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 03:47:32 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[33015]: 2024-05-09T03:47:32.173+0000 7f78af4ed700 -1 osd.6 0 waiting for initial osdmap 2024-05-09T03:47:32.425 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 03:47:32 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[33015]: 2024-05-09T03:47:32.180+0000 7f78a9e84700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T03:47:32.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:32 smithi078 ceph-mon[22905]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:32.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:32 smithi078 ceph-mon[22905]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-09T03:47:32.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:32 smithi078 ceph-mon[22905]: osdmap e44: 7 total, 6 up, 7 in 2024-05-09T03:47:32.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:32 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:32.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:32 smithi078 ceph-mon[22905]: from='osd.6 [v2:172.21.15.132:6816/1013115018,v1:172.21.15.132:6817/1013115018]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:32.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:32 smithi078 ceph-mon[22905]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:32 smithi078 ceph-mon[27629]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:32.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:32 smithi078 ceph-mon[27629]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-09T03:47:32.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:32 smithi078 ceph-mon[27629]: osdmap e44: 7 total, 6 up, 7 in 2024-05-09T03:47:32.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:32 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:32.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:32 smithi078 ceph-mon[27629]: from='osd.6 [v2:172.21.15.132:6816/1013115018,v1:172.21.15.132:6817/1013115018]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:32.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:32 smithi078 ceph-mon[27629]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:33.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:33 smithi132 ceph-mon[23807]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]': finished 2024-05-09T03:47:33.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:33 smithi132 ceph-mon[23807]: osdmap e45: 7 total, 6 up, 7 in 2024-05-09T03:47:33.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:33 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:33.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:33 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:33.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:33 smithi132 ceph-mon[23807]: osd.6 [v2:172.21.15.132:6816/1013115018,v1:172.21.15.132:6817/1013115018] boot 2024-05-09T03:47:33.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:33 smithi132 ceph-mon[23807]: osdmap e46: 7 total, 7 up, 7 in 2024-05-09T03:47:33.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:33 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[22905]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]': finished 2024-05-09T03:47:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[22905]: osdmap e45: 7 total, 6 up, 7 in 2024-05-09T03:47:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[22905]: osd.6 [v2:172.21.15.132:6816/1013115018,v1:172.21.15.132:6817/1013115018] boot 2024-05-09T03:47:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[22905]: osdmap e46: 7 total, 7 up, 7 in 2024-05-09T03:47:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:33.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[27629]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]': finished 2024-05-09T03:47:33.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[27629]: osdmap e45: 7 total, 6 up, 7 in 2024-05-09T03:47:33.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:33.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:33.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[27629]: osd.6 [v2:172.21.15.132:6816/1013115018,v1:172.21.15.132:6817/1013115018] boot 2024-05-09T03:47:33.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[27629]: osdmap e46: 7 total, 7 up, 7 in 2024-05-09T03:47:33.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:33 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:47:34.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:34 smithi132 ceph-mon[23807]: purged_snaps scrub starts 2024-05-09T03:47:34.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:34 smithi132 ceph-mon[23807]: purged_snaps scrub ok 2024-05-09T03:47:34.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:34 smithi132 ceph-mon[23807]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:34.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:34 smithi132 ceph-mon[23807]: osdmap e47: 7 total, 7 up, 7 in 2024-05-09T03:47:34.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:34 smithi078 ceph-mon[22905]: purged_snaps scrub starts 2024-05-09T03:47:34.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:34 smithi078 ceph-mon[22905]: purged_snaps scrub ok 2024-05-09T03:47:34.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:34 smithi078 ceph-mon[22905]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:34.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:34 smithi078 ceph-mon[22905]: osdmap e47: 7 total, 7 up, 7 in 2024-05-09T03:47:34.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:34 smithi078 ceph-mon[27629]: purged_snaps scrub starts 2024-05-09T03:47:34.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:34 smithi078 ceph-mon[27629]: purged_snaps scrub ok 2024-05-09T03:47:34.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:34 smithi078 ceph-mon[27629]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-09T03:47:34.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:34 smithi078 ceph-mon[27629]: osdmap e47: 7 total, 7 up, 7 in 2024-05-09T03:47:36.165 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:35 smithi132 ceph-mon[23807]: pgmap v148: 1 pgs: 1 peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:36.165 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:35 smithi132 ceph-mon[23807]: osdmap e48: 7 total, 7 up, 7 in 2024-05-09T03:47:36.165 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:35 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:36.165 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:35 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:36.165 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:35 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:36.165 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:35 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:36.165 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:35 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:36.165 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:35 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:36.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[22905]: pgmap v148: 1 pgs: 1 peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:36.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[22905]: osdmap e48: 7 total, 7 up, 7 in 2024-05-09T03:47:36.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:36.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:36.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:36.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:36.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:36.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:36.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[27629]: pgmap v148: 1 pgs: 1 peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:36.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[27629]: osdmap e48: 7 total, 7 up, 7 in 2024-05-09T03:47:36.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:36.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:36.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:36.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:47:36.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:36.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:35 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:36.766 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-05-09T03:47:36.786 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi132:vg_nvme/lv_1 2024-05-09T03:47:37.093 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:36 smithi132 ceph-mon[23807]: Detected new or changed devices on smithi132 2024-05-09T03:47:37.093 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:36 smithi132 ceph-mon[23807]: Adjusting osd_memory_target on smithi132 to 5696M 2024-05-09T03:47:37.094 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:36 smithi132 ceph-mon[23807]: osdmap e49: 7 total, 7 up, 7 in 2024-05-09T03:47:37.094 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:36 smithi132 ceph-mon[23807]: osdmap e50: 7 total, 7 up, 7 in 2024-05-09T03:47:37.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:36 smithi078 ceph-mon[22905]: Detected new or changed devices on smithi132 2024-05-09T03:47:37.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:36 smithi078 ceph-mon[22905]: Adjusting osd_memory_target on smithi132 to 5696M 2024-05-09T03:47:37.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:36 smithi078 ceph-mon[22905]: osdmap e49: 7 total, 7 up, 7 in 2024-05-09T03:47:37.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:36 smithi078 ceph-mon[22905]: osdmap e50: 7 total, 7 up, 7 in 2024-05-09T03:47:37.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:36 smithi078 ceph-mon[27629]: Detected new or changed devices on smithi132 2024-05-09T03:47:37.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:36 smithi078 ceph-mon[27629]: Adjusting osd_memory_target on smithi132 to 5696M 2024-05-09T03:47:37.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:36 smithi078 ceph-mon[27629]: osdmap e49: 7 total, 7 up, 7 in 2024-05-09T03:47:37.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:36 smithi078 ceph-mon[27629]: osdmap e50: 7 total, 7 up, 7 in 2024-05-09T03:47:38.060 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:37 smithi132 ceph-mon[23807]: pgmap v151: 1 pgs: 1 peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:38.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:37 smithi078 ceph-mon[22905]: pgmap v151: 1 pgs: 1 peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:38.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:37 smithi078 ceph-mon[27629]: pgmap v151: 1 pgs: 1 peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:39.101 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:38 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:47:39.101 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:38 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:47:39.101 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:38 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:39.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:38 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:47:39.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:38 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:47:39.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:38 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:39.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:38 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-09T03:47:39.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:38 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-09T03:47:39.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:38 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:40.162 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:39 smithi132 ceph-mon[23807]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 114 KiB/s, 0 objects/s recovering 2024-05-09T03:47:40.162 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:39 smithi132 ceph-mon[23807]: from='client.24272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi132:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:47:40.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:39 smithi078 ceph-mon[22905]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 114 KiB/s, 0 objects/s recovering 2024-05-09T03:47:40.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:39 smithi078 ceph-mon[22905]: from='client.24272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi132:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:47:40.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:39 smithi078 ceph-mon[27629]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 114 KiB/s, 0 objects/s recovering 2024-05-09T03:47:40.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:39 smithi078 ceph-mon[27629]: from='client.24272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi132:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:47:42.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:41 smithi132 ceph-mon[23807]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-09T03:47:42.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:41 smithi132 ceph-mon[23807]: from='client.? 172.21.15.132:0/1691878991' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e3c95f34-9aee-4cd5-809f-9068fb32a376"}]: dispatch 2024-05-09T03:47:42.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:41 smithi132 ceph-mon[23807]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e3c95f34-9aee-4cd5-809f-9068fb32a376"}]: dispatch 2024-05-09T03:47:42.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:41 smithi132 ceph-mon[23807]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e3c95f34-9aee-4cd5-809f-9068fb32a376"}]': finished 2024-05-09T03:47:42.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:41 smithi132 ceph-mon[23807]: osdmap e51: 8 total, 7 up, 8 in 2024-05-09T03:47:42.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:41 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:42.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:41 smithi132 ceph-mon[23807]: from='client.? 172.21.15.132:0/3562465516' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:47:42.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[27629]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-09T03:47:42.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[27629]: from='client.? 172.21.15.132:0/1691878991' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e3c95f34-9aee-4cd5-809f-9068fb32a376"}]: dispatch 2024-05-09T03:47:42.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[27629]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e3c95f34-9aee-4cd5-809f-9068fb32a376"}]: dispatch 2024-05-09T03:47:42.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[27629]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e3c95f34-9aee-4cd5-809f-9068fb32a376"}]': finished 2024-05-09T03:47:42.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[27629]: osdmap e51: 8 total, 7 up, 8 in 2024-05-09T03:47:42.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:42.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[27629]: from='client.? 172.21.15.132:0/3562465516' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:47:42.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[22905]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-09T03:47:42.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[22905]: from='client.? 172.21.15.132:0/1691878991' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e3c95f34-9aee-4cd5-809f-9068fb32a376"}]: dispatch 2024-05-09T03:47:42.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[22905]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e3c95f34-9aee-4cd5-809f-9068fb32a376"}]: dispatch 2024-05-09T03:47:42.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[22905]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e3c95f34-9aee-4cd5-809f-9068fb32a376"}]': finished 2024-05-09T03:47:42.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[22905]: osdmap e51: 8 total, 7 up, 8 in 2024-05-09T03:47:42.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:42.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:41 smithi078 ceph-mon[22905]: from='client.? 172.21.15.132:0/3562465516' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-09T03:47:44.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:43 smithi132 ceph-mon[23807]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-05-09T03:47:44.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:43 smithi078 ceph-mon[22905]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-05-09T03:47:44.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:43 smithi078 ceph-mon[27629]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-05-09T03:47:46.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:45 smithi132 ceph-mon[23807]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-09T03:47:46.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:45 smithi078 ceph-mon[22905]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-09T03:47:46.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:45 smithi078 ceph-mon[27629]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-09T03:47:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:47 smithi132 ceph-mon[23807]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-05-09T03:47:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:47 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-09T03:47:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:47 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:48.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:47 smithi078 ceph-mon[22905]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-05-09T03:47:48.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:47 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-09T03:47:48.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:47 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:47 smithi078 ceph-mon[27629]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-05-09T03:47:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:47 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-09T03:47:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:47 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:49.251 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:48 smithi132 ceph-mon[23807]: Deploying daemon osd.7 on smithi132 2024-05-09T03:47:49.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:48 smithi078 ceph-mon[22905]: Deploying daemon osd.7 on smithi132 2024-05-09T03:47:49.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:48 smithi078 ceph-mon[27629]: Deploying daemon osd.7 on smithi132 2024-05-09T03:47:50.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:49 smithi132 ceph-mon[23807]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:50.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:49 smithi078 ceph-mon[27629]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:50.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:49 smithi078 ceph-mon[22905]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:52.241 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:51 smithi132 ceph-mon[23807]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:52.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:51 smithi078 ceph-mon[22905]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:52.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:51 smithi078 ceph-mon[27629]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:53.206 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:47:53.207 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:47:53.207 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:53.207 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:53.207 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:53.207 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:52 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:53.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:47:53.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:47:53.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:53.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:53.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:53.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:52 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:53.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:47:53.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:47:53.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:53.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:53.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:53.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:52 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:54.218 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:53 smithi132 ceph-mon[23807]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:54.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:53 smithi078 ceph-mon[22905]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:54.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:53 smithi078 ceph-mon[27629]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:56.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:55 smithi132 ceph-mon[23807]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:56.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:55 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:56.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:55 smithi132 ceph-mon[23807]: from='osd.7 [v2:172.21.15.132:6824/3753810644,v1:172.21.15.132:6825/3753810644]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-09T03:47:56.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:55 smithi132 ceph-mon[23807]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-09T03:47:56.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:55 smithi078 ceph-mon[22905]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:56.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:55 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:56.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:55 smithi078 ceph-mon[22905]: from='osd.7 [v2:172.21.15.132:6824/3753810644,v1:172.21.15.132:6825/3753810644]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-09T03:47:56.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:55 smithi078 ceph-mon[22905]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-09T03:47:56.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:55 smithi078 ceph-mon[27629]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:56.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:55 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:56.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:55 smithi078 ceph-mon[27629]: from='osd.7 [v2:172.21.15.132:6824/3753810644,v1:172.21.15.132:6825/3753810644]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-09T03:47:56.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:55 smithi078 ceph-mon[27629]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-09T03:47:56.326 INFO:teuthology.orchestra.run.smithi132.stdout:Created osd(s) 7 on host 'smithi132' 2024-05-09T03:47:56.938 DEBUG:teuthology.orchestra.run.smithi132:osd.7> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.7.service 2024-05-09T03:47:56.941 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-05-09T03:47:56.941 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd stat -f json 2024-05-09T03:47:57.225 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:57 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:57.226 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:57 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:57.226 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:57 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:57.226 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:57 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:57.226 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:57 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:57.226 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:57 smithi132 ceph-mon[23807]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-09T03:47:57.226 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:57 smithi132 ceph-mon[23807]: osdmap e52: 8 total, 7 up, 8 in 2024-05-09T03:47:57.226 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:57 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:57.226 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:57 smithi132 ceph-mon[23807]: from='osd.7 [v2:172.21.15.132:6824/3753810644,v1:172.21.15.132:6825/3753810644]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:57.226 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:57 smithi132 ceph-mon[23807]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:57.255 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:57.255 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:57.255 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:57.255 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:57.255 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:57.255 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[22905]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-09T03:47:57.255 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[22905]: osdmap e52: 8 total, 7 up, 8 in 2024-05-09T03:47:57.256 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:57.256 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[22905]: from='osd.7 [v2:172.21.15.132:6824/3753810644,v1:172.21.15.132:6825/3753810644]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:57.256 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[22905]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:57.256 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:57.256 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:47:57.256 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:47:57.256 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:47:57.257 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:47:57.257 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[27629]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-09T03:47:57.257 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[27629]: osdmap e52: 8 total, 7 up, 8 in 2024-05-09T03:47:57.257 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:57.257 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[27629]: from='osd.7 [v2:172.21.15.132:6824/3753810644,v1:172.21.15.132:6825/3753810644]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:57.257 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:57 smithi078 ceph-mon[27629]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T03:47:58.004 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 03:47:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[35931]: 2024-05-09T03:47:57.505+0000 7f2253d59700 -1 osd.7 0 waiting for initial osdmap 2024-05-09T03:47:58.004 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 03:47:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[35931]: 2024-05-09T03:47:57.511+0000 7f224c6ec700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T03:47:58.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:58 smithi132 ceph-mon[23807]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:58.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:58 smithi132 ceph-mon[23807]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]': finished 2024-05-09T03:47:58.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:58 smithi132 ceph-mon[23807]: osdmap e53: 8 total, 7 up, 8 in 2024-05-09T03:47:58.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:58 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:58.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:58 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:58.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:58 smithi132 ceph-mon[23807]: osd.7 [v2:172.21.15.132:6824/3753810644,v1:172.21.15.132:6825/3753810644] boot 2024-05-09T03:47:58.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:58 smithi132 ceph-mon[23807]: osdmap e54: 8 total, 8 up, 8 in 2024-05-09T03:47:58.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:58 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[27629]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[27629]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]': finished 2024-05-09T03:47:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[27629]: osdmap e53: 8 total, 7 up, 8 in 2024-05-09T03:47:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:58.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:58.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[27629]: osd.7 [v2:172.21.15.132:6824/3753810644,v1:172.21.15.132:6825/3753810644] boot 2024-05-09T03:47:58.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[27629]: osdmap e54: 8 total, 8 up, 8 in 2024-05-09T03:47:58.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[22905]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:47:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[22905]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]': finished 2024-05-09T03:47:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[22905]: osdmap e53: 8 total, 7 up, 8 in 2024-05-09T03:47:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[22905]: osd.7 [v2:172.21.15.132:6824/3753810644,v1:172.21.15.132:6825/3753810644] boot 2024-05-09T03:47:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[22905]: osdmap e54: 8 total, 8 up, 8 in 2024-05-09T03:47:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:58 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:47:59.237 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:47:59.992 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1715226477,"num_in_osds":8,"osd_in_since":1715226461,"num_remapped_pgs":1} 2024-05-09T03:47:59.993 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd dump --format=json 2024-05-09T03:48:00.213 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:00.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:59 smithi132 ceph-mon[23807]: purged_snaps scrub starts 2024-05-09T03:48:00.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:59 smithi132 ceph-mon[23807]: purged_snaps scrub ok 2024-05-09T03:48:00.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:59 smithi132 ceph-mon[23807]: pgmap v167: 1 pgs: 1 peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:48:00.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:59 smithi132 ceph-mon[23807]: osdmap e55: 8 total, 8 up, 8 in 2024-05-09T03:48:00.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:47:59 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1994342187' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-09T03:48:00.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:59 smithi078 ceph-mon[22905]: purged_snaps scrub starts 2024-05-09T03:48:00.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:59 smithi078 ceph-mon[22905]: purged_snaps scrub ok 2024-05-09T03:48:00.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:59 smithi078 ceph-mon[22905]: pgmap v167: 1 pgs: 1 peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:48:00.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:59 smithi078 ceph-mon[22905]: osdmap e55: 8 total, 8 up, 8 in 2024-05-09T03:48:00.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:47:59 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1994342187' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-09T03:48:00.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:59 smithi078 ceph-mon[27629]: purged_snaps scrub starts 2024-05-09T03:48:00.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:59 smithi078 ceph-mon[27629]: purged_snaps scrub ok 2024-05-09T03:48:00.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:59 smithi078 ceph-mon[27629]: pgmap v167: 1 pgs: 1 peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-09T03:48:00.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:59 smithi078 ceph-mon[27629]: osdmap e55: 8 total, 8 up, 8 in 2024-05-09T03:48:00.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:47:59 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1994342187' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-09T03:48:01.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:00 smithi132 ceph-mon[23807]: osdmap e56: 8 total, 8 up, 8 in 2024-05-09T03:48:01.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:00 smithi132 ceph-mon[23807]: Detected new or changed devices on smithi132 2024-05-09T03:48:01.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:00 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:01.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:00 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:48:01.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:00 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:48:01.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:00 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:48:01.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:00 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:48:01.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:00 smithi132 ceph-mon[23807]: Adjusting osd_memory_target on smithi132 to 4272M 2024-05-09T03:48:01.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:00 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:01.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:00 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:01.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[22905]: osdmap e56: 8 total, 8 up, 8 in 2024-05-09T03:48:01.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[22905]: Detected new or changed devices on smithi132 2024-05-09T03:48:01.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:01.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:48:01.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:48:01.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:48:01.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:48:01.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[22905]: Adjusting osd_memory_target on smithi132 to 4272M 2024-05-09T03:48:01.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:01.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:01.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[27629]: osdmap e56: 8 total, 8 up, 8 in 2024-05-09T03:48:01.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[27629]: Detected new or changed devices on smithi132 2024-05-09T03:48:01.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:01.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:48:01.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:48:01.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:48:01.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:48:01.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[27629]: Adjusting osd_memory_target on smithi132 to 4272M 2024-05-09T03:48:01.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:01.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:00 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:02.085 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:48:02.085 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":58,"fsid":"2777c79e-0db6-11ef-bc97-c7b262605968","created":"2024-05-09T03:42:58.514807+0000","modified":"2024-05-09T03:48:02.017103+0000","last_up_change":"2024-05-09T03:47:57.916295+0000","last_in_change":"2024-05-09T03:47:41.214196+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-09T03:46:00.707353+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cbcf6ecf-5076-4a63-b6f7-df5e34185ddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":3700260173},{"type":"v1","addr":"172.21.15.78:6803","nonce":3700260173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":3700260173},{"type":"v1","addr":"172.21.15.78:6805","nonce":3700260173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":3700260173},{"type":"v1","addr":"172.21.15.78:6809","nonce":3700260173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":3700260173},{"type":"v1","addr":"172.21.15.78:6807","nonce":3700260173}]},"public_addr":"172.21.15.78:6803/3700260173","cluster_addr":"172.21.15.78:6805/3700260173","heartbeat_back_addr":"172.21.15.78:6809/3700260173","heartbeat_front_addr":"172.21.15.78:6807/3700260173","state":["exists","up"]},{"osd":1,"uuid":"25c3b194-8ee3-41d3-a172-90c263918e62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":2683958178},{"type":"v1","addr":"172.21.15.78:6811","nonce":2683958178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":2683958178},{"type":"v1","addr":"172.21.15.78:6813","nonce":2683958178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":2683958178},{"type":"v1","addr":"172.21.15.78:6817","nonce":2683958178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":2683958178},{"type":"v1","addr":"172.21.15.78:6815","nonce":2683958178}]},"public_addr":"172.21.15.78:6811/2683958178","cluster_addr":"172.21.15.78:6813/2683958178","heartbeat_back_addr":"172.21.15.78:6817/2683958178","heartbeat_front_addr":"172.21.15.78:6815/2683958178","state":["exists","up"]},{"osd":2,"uuid":"86841bbd-3c86-4114-9347-e1b9dbd8358f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":3785529399},{"type":"v1","addr":"172.21.15.78:6819","nonce":3785529399}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":3785529399},{"type":"v1","addr":"172.21.15.78:6821","nonce":3785529399}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":3785529399},{"type":"v1","addr":"172.21.15.78:6825","nonce":3785529399}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":3785529399},{"type":"v1","addr":"172.21.15.78:6823","nonce":3785529399}]},"public_addr":"172.21.15.78:6819/3785529399","cluster_addr":"172.21.15.78:6821/3785529399","heartbeat_back_addr":"172.21.15.78:6825/3785529399","heartbeat_front_addr":"172.21.15.78:6823/3785529399","state":["exists","up"]},{"osd":3,"uuid":"97f610ec-76cf-4cb4-8a9a-f111e8b437c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":3522076329},{"type":"v1","addr":"172.21.15.78:6827","nonce":3522076329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":3522076329},{"type":"v1","addr":"172.21.15.78:6829","nonce":3522076329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":3522076329},{"type":"v1","addr":"172.21.15.78:6833","nonce":3522076329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":3522076329},{"type":"v1","addr":"172.21.15.78:6831","nonce":3522076329}]},"public_addr":"172.21.15.78:6827/3522076329","cluster_addr":"172.21.15.78:6829/3522076329","heartbeat_back_addr":"172.21.15.78:6833/3522076329","heartbeat_front_addr":"172.21.15.78:6831/3522076329","state":["exists","up"]},{"osd":4,"uuid":"bd1bd685-528f-41bd-97fc-5aabda193ce2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6800","nonce":268288079},{"type":"v1","addr":"172.21.15.132:6801","nonce":268288079}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6802","nonce":268288079},{"type":"v1","addr":"172.21.15.132:6803","nonce":268288079}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6806","nonce":268288079},{"type":"v1","addr":"172.21.15.132:6807","nonce":268288079}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6804","nonce":268288079},{"type":"v1","addr":"172.21.15.132:6805","nonce":268288079}]},"public_addr":"172.21.15.132:6801/268288079","cluster_addr":"172.21.15.132:6803/268288079","heartbeat_back_addr":"172.21.15.132:6807/268288079","heartbeat_front_addr":"172.21.15.132:6805/268288079","state":["exists","up"]},{"osd":5,"uuid":"30240d46-0459-4fde-ac42-1c4a82b483ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6808","nonce":968710714},{"type":"v1","addr":"172.21.15.132:6809","nonce":968710714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6810","nonce":968710714},{"type":"v1","addr":"172.21.15.132:6811","nonce":968710714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6814","nonce":968710714},{"type":"v1","addr":"172.21.15.132:6815","nonce":968710714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6812","nonce":968710714},{"type":"v1","addr":"172.21.15.132:6813","nonce":968710714}]},"public_addr":"172.21.15.132:6809/968710714","cluster_addr":"172.21.15.132:6811/968710714","heartbeat_back_addr":"172.21.15.132:6815/968710714","heartbeat_front_addr":"172.21.15.132:6813/968710714","state":["exists","up"]},{"osd":6,"uuid":"d6b0f898-9036-4b02-9cba-08e377b14063","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6816","nonce":1013115018},{"type":"v1","addr":"172.21.15.132:6817","nonce":1013115018}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6818","nonce":1013115018},{"type":"v1","addr":"172.21.15.132:6819","nonce":1013115018}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6822","nonce":1013115018},{"type":"v1","addr":"172.21.15.132:6823","nonce":1013115018}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6820","nonce":1013115018},{"type":"v1","addr":"172.21.15.132:6821","nonce":1013115018}]},"public_addr":"172.21.15.132:6817/1013115018","cluster_addr":"172.21.15.132:6819/1013115018","heartbeat_back_addr":"172.21.15.132:6823/1013115018","heartbeat_front_addr":"172.21.15.132:6821/1013115018","state":["exists","up"]},{"osd":7,"uuid":"e3c95f34-9aee-4cd5-809f-9068fb32a376","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6824","nonce":3753810644},{"type":"v1","addr":"172.21.15.132:6825","nonce":3753810644}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6826","nonce":3753810644},{"type":"v1","addr":"172.21.15.132:6827","nonce":3753810644}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6830","nonce":3753810644},{"type":"v1","addr":"172.21.15.132:6831","nonce":3753810644}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6828","nonce":3753810644},{"type":"v1","addr":"172.21.15.132:6829","nonce":3753810644}]},"public_addr":"172.21.15.132:6825/3753810644","cluster_addr":"172.21.15.132:6827/3753810644","heartbeat_back_addr":"172.21.15.132:6831/3753810644","heartbeat_front_addr":"172.21.15.132:6829/3753810644","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:45:09.956224+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:45:34.301481+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:45:57.790210+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:46:21.497210+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:46:43.185935+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:47:06.429857+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:47:31.280857+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:47:56.922326+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/3439374234":"2024-05-10T03:43:52.360677+0000","172.21.15.78:6801/987239519":"2024-05-10T03:43:52.360677+0000","172.21.15.78:6800/987239519":"2024-05-10T03:43:52.360677+0000","172.21.15.78:6800/169193258":"2024-05-10T03:43:28.859080+0000","172.21.15.78:6801/169193258":"2024-05-10T03:43:28.859080+0000","172.21.15.78:0/3754415003":"2024-05-10T03:43:52.360677+0000","172.21.15.78:0/906720789":"2024-05-10T03:43:52.360677+0000","172.21.15.78:0/2700265794":"2024-05-10T03:43:28.859080+0000","172.21.15.78:0/147802961":"2024-05-10T03:43:28.859080+0000","172.21.15.78:0/3364333504":"2024-05-10T03:43:28.859080+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-09T03:48:02.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:02 smithi078 ceph-mon[22905]: pgmap v170: 1 pgs: 1 peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:02.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:02 smithi078 ceph-mon[22905]: osdmap e57: 8 total, 8 up, 8 in 2024-05-09T03:48:02.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:02 smithi078 ceph-mon[27629]: pgmap v170: 1 pgs: 1 peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:02.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:02 smithi078 ceph-mon[27629]: osdmap e57: 8 total, 8 up, 8 in 2024-05-09T03:48:02.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:02 smithi132 ceph-mon[23807]: pgmap v170: 1 pgs: 1 peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:02.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:02 smithi132 ceph-mon[23807]: osdmap e57: 8 total, 8 up, 8 in 2024-05-09T03:48:02.725 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-05-09T03:46:00.707353+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-05-09T03:48:02.726 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-05-09T03:48:02.937 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:03.281 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:03 smithi078 ceph-mon[27629]: osdmap e58: 8 total, 8 up, 8 in 2024-05-09T03:48:03.281 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:03 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/4180611434' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-09T03:48:03.281 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:03 smithi078 ceph-mon[22905]: osdmap e58: 8 total, 8 up, 8 in 2024-05-09T03:48:03.281 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:03 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/4180611434' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-09T03:48:03.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:03 smithi132 ceph-mon[23807]: osdmap e58: 8 total, 8 up, 8 in 2024-05-09T03:48:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:03 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/4180611434' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-09T03:48:04.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:04 smithi078 ceph-mon[22905]: pgmap v173: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:04.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:04 smithi078 ceph-mon[27629]: pgmap v173: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:04.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:04 smithi132 ceph-mon[23807]: pgmap v173: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:04.679 INFO:teuthology.orchestra.run.smithi078.stdout:pg_num: 1 2024-05-09T03:48:05.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:05 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3652071095' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-09T03:48:05.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:05 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3652071095' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-09T03:48:05.415 INFO:tasks.cephadm:Adding prometheus.a on smithi132 2024-05-09T03:48:05.415 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch apply prometheus '1;smithi132=a' 2024-05-09T03:48:05.445 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:05 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3652071095' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-09T03:48:06.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:06 smithi132 ceph-mon[23807]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-09T03:48:06.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:06 smithi078 ceph-mon[22905]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-09T03:48:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:06 smithi078 ceph-mon[27629]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-09T03:48:07.375 INFO:teuthology.orchestra.run.smithi132.stdout:Scheduled prometheus update... 2024-05-09T03:48:07.992 DEBUG:teuthology.orchestra.run.smithi132:prometheus.a> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service 2024-05-09T03:48:08.036 INFO:tasks.cephadm:Adding node-exporter.a on smithi078 2024-05-09T03:48:08.036 INFO:tasks.cephadm:Adding node-exporter.b on smithi132 2024-05-09T03:48:08.036 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch apply node-exporter '2;smithi078=a;smithi132=b' 2024-05-09T03:48:08.314 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[22905]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-05-09T03:48:08.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:08.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:48:08.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:48:08.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:48:08.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:08.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-09T03:48:08.315 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[27629]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-05-09T03:48:08.316 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:08.316 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:48:08.316 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:48:08.316 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:48:08.316 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:08.316 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-09T03:48:08.377 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:08 smithi132 ceph-mon[23807]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-05-09T03:48:08.377 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:08 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:08.377 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:08 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:48:08.378 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:08 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:48:08.378 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:08 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:48:08.378 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:08 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' 2024-05-09T03:48:08.378 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:08 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-09T03:48:08.601 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:08 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ignoring --setuser ceph since I am not root 2024-05-09T03:48:08.601 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:08 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ignoring --setgroup ceph since I am not root 2024-05-09T03:48:08.601 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:08 smithi078 ceph-mgr[23144]: -- 172.21.15.78:0/1702700038 <== mon.1 v2:172.21.15.78:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ac72d70340 con 0x55ac72d49000 2024-05-09T03:48:08.601 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:08 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:08.600+0000 7f807a495000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-09T03:48:08.631 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:08 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ignoring --setuser ceph since I am not root 2024-05-09T03:48:08.631 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:08 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ignoring --setgroup ceph since I am not root 2024-05-09T03:48:08.631 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:08 smithi132 ceph-mgr[24993]: -- 172.21.15.132:0/2726953108 <== mon.2 v2:172.21.15.132:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557b92aec340 con 0x557b92ac5000 2024-05-09T03:48:08.631 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:08 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:08.593+0000 7f66026c8000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-09T03:48:09.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:08 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:08.726+0000 7f66026c8000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:48:09.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:08 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:08.825+0000 7f66026c8000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-09T03:48:09.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:08 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:08.726+0000 7f807a495000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:48:09.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:08 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:08.824+0000 7f807a495000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-09T03:48:09.338 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:09 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:09.138+0000 7f66026c8000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-09T03:48:09.338 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:09 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:09.220+0000 7f66026c8000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-09T03:48:09.338 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:09 smithi132 ceph-mon[23807]: from='client.24305 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi132=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:48:09.339 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:09 smithi132 ceph-mon[23807]: Saving service prometheus spec with placement smithi132=a;count:1 2024-05-09T03:48:09.339 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:09 smithi132 ceph-mon[23807]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-09T03:48:09.339 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:09 smithi132 ceph-mon[23807]: mgrmap e16: y(active, since 4m), standbys: x 2024-05-09T03:48:09.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:09 smithi078 ceph-mon[22905]: from='client.24305 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi132=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:48:09.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:09 smithi078 ceph-mon[22905]: Saving service prometheus spec with placement smithi132=a;count:1 2024-05-09T03:48:09.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:09 smithi078 ceph-mon[22905]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-09T03:48:09.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:09 smithi078 ceph-mon[22905]: mgrmap e16: y(active, since 4m), standbys: x 2024-05-09T03:48:09.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:09.129+0000 7f807a495000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-09T03:48:09.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:09.212+0000 7f807a495000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-09T03:48:09.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:09 smithi078 ceph-mon[27629]: from='client.24305 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi132=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:48:09.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:09 smithi078 ceph-mon[27629]: Saving service prometheus spec with placement smithi132=a;count:1 2024-05-09T03:48:09.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:09 smithi078 ceph-mon[27629]: from='mgr.14150 172.21.15.78:0/2420035112' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-09T03:48:09.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:09 smithi078 ceph-mon[27629]: mgrmap e16: y(active, since 4m), standbys: x 2024-05-09T03:48:10.753 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:10 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:10.388+0000 7f66026c8000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-09T03:48:10.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:10 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:10.552+0000 7f66026c8000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-09T03:48:10.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:10 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:10.635+0000 7f66026c8000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-09T03:48:10.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:10.324+0000 7f807a495000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-09T03:48:10.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:10.487+0000 7f807a495000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-09T03:48:10.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:10.568+0000 7f807a495000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-09T03:48:11.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:10 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:10.948+0000 7f66026c8000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-09T03:48:11.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:11 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:11.039+0000 7f66026c8000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-09T03:48:11.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:11 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:11.132+0000 7f66026c8000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-09T03:48:11.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:10.879+0000 7f807a495000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-09T03:48:11.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:10.969+0000 7f807a495000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-09T03:48:11.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:11 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:11.061+0000 7f807a495000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-09T03:48:11.741 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:11 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:11.302+0000 7f807a495000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:48:11.753 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:11 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:11.375+0000 7f66026c8000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:48:12.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:11 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:11.740+0000 7f807a495000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-09T03:48:12.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:11 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:11.815+0000 7f66026c8000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-09T03:48:12.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:12.225+0000 7f807a495000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-09T03:48:12.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:12.319+0000 7f807a495000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-09T03:48:12.687 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:12 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:12.301+0000 7f66026c8000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-09T03:48:12.687 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:12 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:12.394+0000 7f66026c8000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-09T03:48:13.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:12 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:12.687+0000 7f66026c8000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-09T03:48:13.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:12 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:12.775+0000 7f66026c8000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-09T03:48:13.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:12.612+0000 7f807a495000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-09T03:48:13.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:12.701+0000 7f807a495000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-09T03:48:13.444 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:13.104+0000 7f807a495000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-09T03:48:13.444 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:13.237+0000 7f807a495000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-09T03:48:13.445 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:13.330+0000 7f807a495000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-09T03:48:13.445 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:13.443+0000 7f807a495000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-09T03:48:13.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:13.187+0000 7f66026c8000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-09T03:48:13.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:13.328+0000 7f66026c8000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-09T03:48:13.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:13.420+0000 7f66026c8000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-09T03:48:13.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:13.533+0000 7f807a495000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-09T03:48:14.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:13.535+0000 7f66026c8000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-09T03:48:14.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:13.625+0000 7f66026c8000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-09T03:48:14.436 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:14 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:14.076+0000 7f807a495000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-09T03:48:14.498 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:14.171+0000 7f66026c8000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-09T03:48:14.697 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:14 smithi078 ceph-mon[27629]: Active manager daemon y restarted 2024-05-09T03:48:14.697 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:14 smithi078 ceph-mon[27629]: Activating manager daemon y 2024-05-09T03:48:14.697 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:14 smithi078 ceph-mon[27629]: osdmap e59: 8 total, 8 up, 8 in 2024-05-09T03:48:14.698 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:14 smithi078 ceph-mon[22905]: Active manager daemon y restarted 2024-05-09T03:48:14.698 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:14 smithi078 ceph-mon[22905]: Activating manager daemon y 2024-05-09T03:48:14.698 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:14 smithi078 ceph-mon[22905]: osdmap e59: 8 total, 8 up, 8 in 2024-05-09T03:48:14.698 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:14 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:14.435+0000 7f807a495000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-09T03:48:14.698 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:14 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:48:14] ENGINE Bus STARTING 2024-05-09T03:48:14.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:14 smithi132 ceph-mon[23807]: Active manager daemon y restarted 2024-05-09T03:48:14.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:14 smithi132 ceph-mon[23807]: Activating manager daemon y 2024-05-09T03:48:14.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:14 smithi132 ceph-mon[23807]: osdmap e59: 8 total, 8 up, 8 in 2024-05-09T03:48:14.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:14.538+0000 7f66026c8000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-09T03:48:14.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: [09/May/2024:03:48:14] ENGINE Bus STARTING 2024-05-09T03:48:14.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: CherryPy Checker: 2024-05-09T03:48:14.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: The Application mounted at '' has an empty config. 2024-05-09T03:48:14.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: 2024-05-09T03:48:14.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: [09/May/2024:03:48:14] ENGINE Serving on http://:::9283 2024-05-09T03:48:14.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: [09/May/2024:03:48:14] ENGINE Bus STARTED 2024-05-09T03:48:15.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:14 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: CherryPy Checker: 2024-05-09T03:48:15.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:14 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: The Application mounted at '' has an empty config. 2024-05-09T03:48:15.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:14 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:48:15.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:14 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:48:14] ENGINE Serving on http://:::9283 2024-05-09T03:48:15.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:14 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:48:14] ENGINE Bus STARTED 2024-05-09T03:48:15.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:14 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:48:14] ENGINE Bus STARTING 2024-05-09T03:48:15.418 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:15 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:48:15] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T03:48:15.418 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:15 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:48:15] ENGINE Bus STARTED 2024-05-09T03:48:15.522 INFO:teuthology.orchestra.run.smithi132.stdout:Scheduled node-exporter update... 2024-05-09T03:48:15.664 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: mgrmap e17: y(active, starting, since 0.0572822s), standbys: x 2024-05-09T03:48:15.664 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:48:15.664 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:48:15.664 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T03:48:15.665 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: Manager daemon y is now available 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: Standby manager daemon x restarted 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: Standby manager daemon x started 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/3107424749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/3107424749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/3107424749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/3107424749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:48:15.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:48:15.667 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:48:15.667 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:48:15.667 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: [09/May/2024:03:48:14] ENGINE Bus STARTING 2024-05-09T03:48:15.667 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: [09/May/2024:03:48:15] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T03:48:15.667 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: [09/May/2024:03:48:15] ENGINE Bus STARTED 2024-05-09T03:48:15.667 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:15.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: mgrmap e17: y(active, starting, since 0.0572822s), standbys: x 2024-05-09T03:48:15.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:48:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:48:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:48:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T03:48:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T03:48:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:48:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:48:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:48:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:48:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:48:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:48:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:48:15.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:48:15.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T03:48:15.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T03:48:15.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T03:48:15.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: Manager daemon y is now available 2024-05-09T03:48:15.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:48:15.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:48:15.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:48:15.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:48:15.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: Standby manager daemon x restarted 2024-05-09T03:48:15.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: Standby manager daemon x started 2024-05-09T03:48:15.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:15.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/3107424749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T03:48:15.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/3107424749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T03:48:15.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/3107424749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T03:48:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: mgrmap e17: y(active, starting, since 0.0572822s), standbys: x 2024-05-09T03:48:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:48:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T03:48:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: Manager daemon y is now available 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: Standby manager daemon x restarted 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: Standby manager daemon x started 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/3107424749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/3107424749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/3107424749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/3107424749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:48:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:48:15.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:48:15.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: [09/May/2024:03:48:14] ENGINE Bus STARTING 2024-05-09T03:48:15.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: [09/May/2024:03:48:15] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T03:48:15.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: [09/May/2024:03:48:15] ENGINE Bus STARTED 2024-05-09T03:48:15.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/3107424749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T03:48:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:48:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:48:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:48:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:48:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: [09/May/2024:03:48:14] ENGINE Bus STARTING 2024-05-09T03:48:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: [09/May/2024:03:48:15] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T03:48:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: [09/May/2024:03:48:15] ENGINE Bus STARTED 2024-05-09T03:48:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:16.262 DEBUG:teuthology.orchestra.run.smithi078:node-exporter.a> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.a.service 2024-05-09T03:48:16.265 DEBUG:teuthology.orchestra.run.smithi132:node-exporter.b> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.b.service 2024-05-09T03:48:16.267 INFO:tasks.cephadm:Adding alertmanager.a on smithi078 2024-05-09T03:48:16.268 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch apply alertmanager '1;smithi078=a' 2024-05-09T03:48:16.748 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[22905]: mgrmap e18: y(active, since 1.07218s), standbys: x 2024-05-09T03:48:16.748 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[22905]: from='client.24323 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi078=a;smithi132=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:48:16.748 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[22905]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:16.748 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[22905]: Saving service node-exporter spec with placement smithi078=a;smithi132=b;count:2 2024-05-09T03:48:16.748 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:16.748 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:16.749 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:16.749 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T03:48:16.749 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[27629]: mgrmap e18: y(active, since 1.07218s), standbys: x 2024-05-09T03:48:16.749 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[27629]: from='client.24323 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi078=a;smithi132=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:48:16.749 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[27629]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:16.749 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[27629]: Saving service node-exporter spec with placement smithi078=a;smithi132=b;count:2 2024-05-09T03:48:16.749 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:16.749 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:16.749 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:16.749 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:16 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T03:48:16.776 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:16 smithi132 ceph-mon[23807]: mgrmap e18: y(active, since 1.07218s), standbys: x 2024-05-09T03:48:16.776 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:16 smithi132 ceph-mon[23807]: from='client.24323 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi078=a;smithi132=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:48:16.776 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:16 smithi132 ceph-mon[23807]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:16.776 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:16 smithi132 ceph-mon[23807]: Saving service node-exporter spec with placement smithi078=a;smithi132=b;count:2 2024-05-09T03:48:16.776 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:16 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:16.776 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:16 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:16.776 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:16 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:16.777 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:16 smithi132 ceph-mon[23807]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T03:48:17.614 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:17 smithi132 ceph-mon[23807]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:17.614 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:17 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:17.615 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:17 smithi132 ceph-mon[23807]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:48:17.615 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:17 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:17.615 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:17 smithi132 ceph-mon[23807]: mgrmap e19: y(active, since 2s), standbys: x 2024-05-09T03:48:17.615 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:17 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:17.615 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:17 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T03:48:18.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[27629]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:18.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:18.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:48:18.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:18.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[27629]: mgrmap e19: y(active, since 2s), standbys: x 2024-05-09T03:48:18.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:18.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T03:48:18.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[22905]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:18.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:18.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:48:18.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:18.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[22905]: mgrmap e19: y(active, since 2s), standbys: x 2024-05-09T03:48:18.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:18.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:17 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T03:48:18.611 INFO:teuthology.orchestra.run.smithi132.stdout:Scheduled alertmanager update... 2024-05-09T03:48:18.969 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:18 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:48:18.970 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:18 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:18.970 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:18 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:18.970 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:18 smithi132 ceph-mon[23807]: Deploying daemon prometheus.a on smithi132 2024-05-09T03:48:18.970 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:18 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:19.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:18 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:48:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:18 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:18 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:18 smithi078 ceph-mon[22905]: Deploying daemon prometheus.a on smithi132 2024-05-09T03:48:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:18 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:19.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:18 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:48:19.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:18 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:19.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:18 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:19.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:18 smithi078 ceph-mon[27629]: Deploying daemon prometheus.a on smithi132 2024-05-09T03:48:19.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:18 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:19.261 DEBUG:teuthology.orchestra.run.smithi078:alertmanager.a> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@alertmanager.a.service 2024-05-09T03:48:19.264 INFO:tasks.cephadm:Adding grafana.a on smithi132 2024-05-09T03:48:19.264 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph orch apply grafana '1;smithi132=a' 2024-05-09T03:48:20.218 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:19 smithi132 ceph-mon[23807]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:20.218 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:19 smithi132 ceph-mon[23807]: from='client.24344 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi078=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:48:20.218 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:19 smithi132 ceph-mon[23807]: Saving service alertmanager spec with placement smithi078=a;count:1 2024-05-09T03:48:20.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:19 smithi078 ceph-mon[22905]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:20.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:19 smithi078 ceph-mon[22905]: from='client.24344 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi078=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:48:20.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:19 smithi078 ceph-mon[22905]: Saving service alertmanager spec with placement smithi078=a;count:1 2024-05-09T03:48:20.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:19 smithi078 ceph-mon[27629]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:20.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:19 smithi078 ceph-mon[27629]: from='client.24344 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi078=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:48:20.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:19 smithi078 ceph-mon[27629]: Saving service alertmanager spec with placement smithi078=a;count:1 2024-05-09T03:48:22.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:21 smithi132 ceph-mon[23807]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:22.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:21 smithi078 ceph-mon[22905]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:22.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:21 smithi078 ceph-mon[27629]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:23.521 INFO:teuthology.orchestra.run.smithi132.stdout:Scheduled grafana update... 2024-05-09T03:48:24.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:23 smithi132 ceph-mon[23807]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:24.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:23 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:24.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:23 smithi078 ceph-mon[22905]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:24.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:23 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:24.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:23 smithi078 ceph-mon[27629]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:24.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:23 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:25.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:24 smithi132 ceph-mon[23807]: from='client.24350 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi132=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:48:25.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:24 smithi132 ceph-mon[23807]: Saving service grafana spec with placement smithi132=a;count:1 2024-05-09T03:48:25.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:24 smithi078 ceph-mon[22905]: from='client.24350 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi132=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:48:25.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:24 smithi078 ceph-mon[22905]: Saving service grafana spec with placement smithi132=a;count:1 2024-05-09T03:48:25.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:24 smithi078 ceph-mon[27629]: from='client.24350 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi132=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:48:25.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:24 smithi078 ceph-mon[27629]: Saving service grafana spec with placement smithi132=a;count:1 2024-05-09T03:48:25.363 DEBUG:teuthology.orchestra.run.smithi132:grafana.a> sudo journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@grafana.a.service 2024-05-09T03:48:25.368 INFO:tasks.cephadm:Setting up client nodes... 2024-05-09T03:48:25.368 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-05-09T03:48:26.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:25 smithi132 ceph-mon[23807]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:26.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:25 smithi078 ceph-mon[27629]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:26.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:25 smithi078 ceph-mon[22905]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:27.655 INFO:teuthology.orchestra.run.smithi078.stdout:[client.0] 2024-05-09T03:48:27.655 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQCLRzxm7HTNJhAAFZTPHBKtfk0fy7rhjfbp2w== 2024-05-09T03:48:28.054 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:27 smithi132 ceph-mon[23807]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:28.054 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:27 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2277953292' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-09T03:48:28.054 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:27 smithi132 ceph-mon[23807]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-09T03:48:28.054 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:27 smithi132 ceph-mon[23807]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-09T03:48:28.244 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:27 smithi078 ceph-mon[22905]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:28.244 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:27 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2277953292' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-09T03:48:28.245 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:27 smithi078 ceph-mon[22905]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-09T03:48:28.245 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:27 smithi078 ceph-mon[22905]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-09T03:48:28.245 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:27 smithi078 ceph-mon[27629]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:28.245 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:27 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2277953292' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-09T03:48:28.245 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:27 smithi078 ceph-mon[27629]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-09T03:48:28.245 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:27 smithi078 ceph-mon[27629]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-09T03:48:28.269 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-09T03:48:28.269 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-05-09T03:48:28.269 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-05-09T03:48:28.307 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-05-09T03:48:30.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:29 smithi132 ceph-mon[23807]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:30.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:29 smithi078 ceph-mon[22905]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:30.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:29 smithi078 ceph-mon[27629]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:30.791 INFO:teuthology.orchestra.run.smithi132.stdout:[client.1] 2024-05-09T03:48:30.791 INFO:teuthology.orchestra.run.smithi132.stdout: key = AQCORzxmKg/nLhAA9bQR16RXn4SuFiz5VOypuw== 2024-05-09T03:48:30.913 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 podman[38844]: 2024-05-09 03:48:30.912911724 +0000 UTC m=+0.834632717 container init c9092d8a07e890fc497ebf547a4d39d2dbe86c2121a10129df1374d169ca06db (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:48:31.254 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 podman[38844]: 2024-05-09 03:48:30.915789995 +0000 UTC m=+0.837510981 container start c9092d8a07e890fc497ebf547a4d39d2dbe86c2121a10129df1374d169ca06db (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:48:31.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.938Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-09T03:48:31.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.938Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-09T03:48:31.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.938Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-09T03:48:31.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.938Z caller=main.go:518 level=info host_details="(Linux 5.14.0-444.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024 x86_64 smithi132 (none))" 2024-05-09T03:48:31.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.938Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-09T03:48:31.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.938Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-09T03:48:31.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.939Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-09T03:48:31.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.940Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-09T03:48:31.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.940Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-09T03:48:31.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.942Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-09T03:48:31.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.942Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.354µs 2024-05-09T03:48:31.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.942Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-09T03:48:31.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.943Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-05-09T03:48:31.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.943Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.352µs wal_replay_duration=396.277µs total_replay_duration=509.298µs 2024-05-09T03:48:31.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.944Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-09T03:48:31.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.944Z caller=main.go:947 level=info msg="TSDB started" 2024-05-09T03:48:31.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.944Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-09T03:48:31.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.960Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.007937ms db_storage=720ns remote_storage=1.589µs web_handler=238ns query_engine=524ns scrape=367.861µs scrape_sd=16.511µs notify=833ns notify_sd=1.53µs rules=15.437855ms 2024-05-09T03:48:31.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:48:30.960Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-09T03:48:31.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 bash[38844]: c9092d8a07e890fc497ebf547a4d39d2dbe86c2121a10129df1374d169ca06db 2024-05-09T03:48:31.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:48:30 smithi132 systemd[1]: Started Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:48:31.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:30 smithi132 ceph-mon[23807]: from='client.? 172.21.15.132:0/1011014758' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-09T03:48:31.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:30 smithi132 ceph-mon[23807]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-09T03:48:31.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:30 smithi132 ceph-mon[23807]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-09T03:48:31.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:30 smithi078 ceph-mon[27629]: from='client.? 172.21.15.132:0/1011014758' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-09T03:48:31.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:30 smithi078 ceph-mon[27629]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-09T03:48:31.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:30 smithi078 ceph-mon[27629]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-09T03:48:31.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:30 smithi078 ceph-mon[22905]: from='client.? 172.21.15.132:0/1011014758' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-09T03:48:31.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:30 smithi078 ceph-mon[22905]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-09T03:48:31.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:30 smithi078 ceph-mon[22905]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-09T03:48:32.012 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:31 smithi078 bash[42240]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-05-09T03:48:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:32 smithi132 ceph-mon[23807]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:32 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:32 smithi132 ceph-mon[23807]: Deploying daemon node-exporter.a on smithi078 2024-05-09T03:48:32.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:32 smithi078 ceph-mon[22905]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:32.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:32 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:32.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:32 smithi078 ceph-mon[22905]: Deploying daemon node-exporter.a on smithi078 2024-05-09T03:48:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:32 smithi078 ceph-mon[27629]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:32 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:32 smithi078 ceph-mon[27629]: Deploying daemon node-exporter.a on smithi078 2024-05-09T03:48:33.068 DEBUG:teuthology.orchestra.run.smithi132:> set -ex 2024-05-09T03:48:33.068 DEBUG:teuthology.orchestra.run.smithi132:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-05-09T03:48:33.068 DEBUG:teuthology.orchestra.run.smithi132:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-05-09T03:48:33.110 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-05-09T03:48:33.110 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-05-09T03:48:33.111 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph mgr dump --format=json 2024-05-09T03:48:33.338 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:33.461 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:33 smithi078 bash[42240]: Getting image source signatures 2024-05-09T03:48:33.461 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:33 smithi078 bash[42240]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-05-09T03:48:33.461 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:33 smithi078 bash[42240]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-05-09T03:48:33.461 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:33 smithi078 bash[42240]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-05-09T03:48:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:34 smithi132 ceph-mon[23807]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:34.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:34 smithi078 ceph-mon[22905]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:34.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:34 smithi078 ceph-mon[27629]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:35.480 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:35 smithi078 bash[42240]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-05-09T03:48:35.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:35 smithi078 ceph-mon[22905]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:35.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:35 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:35.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:35 smithi078 ceph-mon[27629]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:35.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:35 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:35.762 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:35 smithi078 bash[42240]: Writing manifest to image destination 2024-05-09T03:48:35.996 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:48:36.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:35 smithi132 ceph-mon[23807]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:35 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:36.134 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:35 smithi078 podman[42240]: 2024-05-09 03:48:35.928093844 +0000 UTC m=+4.129944204 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-09T03:48:36.134 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 podman[42240]: 2024-05-09 03:48:36.034038615 +0000 UTC m=+4.235888952 container create 10d0785d9bf451416a9ec8973c6fa5291265276fc02dfc788d28eb5e32432bde (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T03:48:36.721 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:36 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2780160790' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-09T03:48:36.721 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2780160790' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-09T03:48:36.722 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 podman[42240]: 2024-05-09 03:48:36.579617874 +0000 UTC m=+4.781468213 container init 10d0785d9bf451416a9ec8973c6fa5291265276fc02dfc788d28eb5e32432bde (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T03:48:36.722 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 podman[42240]: 2024-05-09 03:48:36.582873232 +0000 UTC m=+4.784723571 container start 10d0785d9bf451416a9ec8973c6fa5291265276fc02dfc788d28eb5e32432bde (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T03:48:36.722 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.586Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-09T03:48:36.722 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.586Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-09T03:48:36.722 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-09T03:48:36.722 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-09T03:48:36.722 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-09T03:48:36.722 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=arp 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=edac 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-09T03:48:36.723 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=os 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-09T03:48:36.724 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.587Z caller=node_exporter.go:115 level=info collector=stat 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.588Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.588Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.588Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.588Z caller=node_exporter.go:115 level=info collector=time 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.588Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.588Z caller=node_exporter.go:115 level=info collector=uname 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.588Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.588Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.588Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.588Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-09T03:48:36.725 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[42431]: ts=2024-05-09T03:48:36.588Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-09T03:48:36.726 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 bash[42240]: 10d0785d9bf451416a9ec8973c6fa5291265276fc02dfc788d28eb5e32432bde 2024-05-09T03:48:36.726 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:48:36 smithi078 systemd[1]: Started Ceph node-exporter.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:48:37.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:36 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2780160790' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-09T03:48:37.173 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":19,"active_gid":24340,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":1359853598},{"type":"v1","addr":"172.21.15.78:6801","nonce":1359853598}]},"active_addr":"172.21.15.78:6801/1359853598","active_change":"2024-05-09T03:48:14.438726+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24311,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.78:8443/","prometheus":"http://172.21.15.78:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":59,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":460728655}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":330313505}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":477924627}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":3263238082}]}]}} 2024-05-09T03:48:37.177 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-05-09T03:48:37.177 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-05-09T03:48:37.177 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd dump --format=json 2024-05-09T03:48:37.391 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:37.731 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:37 smithi132 bash[39162]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-05-09T03:48:37.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:37 smithi078 ceph-mon[22905]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:37.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:37 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:37.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:37 smithi078 ceph-mon[22905]: Deploying daemon node-exporter.b on smithi132 2024-05-09T03:48:37.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:37 smithi078 ceph-mon[27629]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:37.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:37 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:37.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:37 smithi078 ceph-mon[27629]: Deploying daemon node-exporter.b on smithi132 2024-05-09T03:48:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:37 smithi132 ceph-mon[23807]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:37 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:37 smithi132 ceph-mon[23807]: Deploying daemon node-exporter.b on smithi132 2024-05-09T03:48:38.004 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:37 smithi132 bash[39162]: Getting image source signatures 2024-05-09T03:48:38.004 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:37 smithi132 bash[39162]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-05-09T03:48:38.004 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:37 smithi132 bash[39162]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-05-09T03:48:38.005 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:37 smithi132 bash[39162]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-05-09T03:48:39.359 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:48:39.360 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":59,"fsid":"2777c79e-0db6-11ef-bc97-c7b262605968","created":"2024-05-09T03:42:58.514807+0000","modified":"2024-05-09T03:48:14.438385+0000","last_up_change":"2024-05-09T03:47:57.916295+0000","last_in_change":"2024-05-09T03:47:41.214196+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-09T03:46:00.707353+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cbcf6ecf-5076-4a63-b6f7-df5e34185ddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":3700260173},{"type":"v1","addr":"172.21.15.78:6803","nonce":3700260173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":3700260173},{"type":"v1","addr":"172.21.15.78:6805","nonce":3700260173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":3700260173},{"type":"v1","addr":"172.21.15.78:6809","nonce":3700260173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":3700260173},{"type":"v1","addr":"172.21.15.78:6807","nonce":3700260173}]},"public_addr":"172.21.15.78:6803/3700260173","cluster_addr":"172.21.15.78:6805/3700260173","heartbeat_back_addr":"172.21.15.78:6809/3700260173","heartbeat_front_addr":"172.21.15.78:6807/3700260173","state":["exists","up"]},{"osd":1,"uuid":"25c3b194-8ee3-41d3-a172-90c263918e62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":2683958178},{"type":"v1","addr":"172.21.15.78:6811","nonce":2683958178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":2683958178},{"type":"v1","addr":"172.21.15.78:6813","nonce":2683958178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":2683958178},{"type":"v1","addr":"172.21.15.78:6817","nonce":2683958178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":2683958178},{"type":"v1","addr":"172.21.15.78:6815","nonce":2683958178}]},"public_addr":"172.21.15.78:6811/2683958178","cluster_addr":"172.21.15.78:6813/2683958178","heartbeat_back_addr":"172.21.15.78:6817/2683958178","heartbeat_front_addr":"172.21.15.78:6815/2683958178","state":["exists","up"]},{"osd":2,"uuid":"86841bbd-3c86-4114-9347-e1b9dbd8358f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":3785529399},{"type":"v1","addr":"172.21.15.78:6819","nonce":3785529399}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":3785529399},{"type":"v1","addr":"172.21.15.78:6821","nonce":3785529399}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":3785529399},{"type":"v1","addr":"172.21.15.78:6825","nonce":3785529399}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":3785529399},{"type":"v1","addr":"172.21.15.78:6823","nonce":3785529399}]},"public_addr":"172.21.15.78:6819/3785529399","cluster_addr":"172.21.15.78:6821/3785529399","heartbeat_back_addr":"172.21.15.78:6825/3785529399","heartbeat_front_addr":"172.21.15.78:6823/3785529399","state":["exists","up"]},{"osd":3,"uuid":"97f610ec-76cf-4cb4-8a9a-f111e8b437c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":3522076329},{"type":"v1","addr":"172.21.15.78:6827","nonce":3522076329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":3522076329},{"type":"v1","addr":"172.21.15.78:6829","nonce":3522076329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":3522076329},{"type":"v1","addr":"172.21.15.78:6833","nonce":3522076329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":3522076329},{"type":"v1","addr":"172.21.15.78:6831","nonce":3522076329}]},"public_addr":"172.21.15.78:6827/3522076329","cluster_addr":"172.21.15.78:6829/3522076329","heartbeat_back_addr":"172.21.15.78:6833/3522076329","heartbeat_front_addr":"172.21.15.78:6831/3522076329","state":["exists","up"]},{"osd":4,"uuid":"bd1bd685-528f-41bd-97fc-5aabda193ce2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6800","nonce":268288079},{"type":"v1","addr":"172.21.15.132:6801","nonce":268288079}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6802","nonce":268288079},{"type":"v1","addr":"172.21.15.132:6803","nonce":268288079}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6806","nonce":268288079},{"type":"v1","addr":"172.21.15.132:6807","nonce":268288079}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6804","nonce":268288079},{"type":"v1","addr":"172.21.15.132:6805","nonce":268288079}]},"public_addr":"172.21.15.132:6801/268288079","cluster_addr":"172.21.15.132:6803/268288079","heartbeat_back_addr":"172.21.15.132:6807/268288079","heartbeat_front_addr":"172.21.15.132:6805/268288079","state":["exists","up"]},{"osd":5,"uuid":"30240d46-0459-4fde-ac42-1c4a82b483ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6808","nonce":968710714},{"type":"v1","addr":"172.21.15.132:6809","nonce":968710714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6810","nonce":968710714},{"type":"v1","addr":"172.21.15.132:6811","nonce":968710714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6814","nonce":968710714},{"type":"v1","addr":"172.21.15.132:6815","nonce":968710714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6812","nonce":968710714},{"type":"v1","addr":"172.21.15.132:6813","nonce":968710714}]},"public_addr":"172.21.15.132:6809/968710714","cluster_addr":"172.21.15.132:6811/968710714","heartbeat_back_addr":"172.21.15.132:6815/968710714","heartbeat_front_addr":"172.21.15.132:6813/968710714","state":["exists","up"]},{"osd":6,"uuid":"d6b0f898-9036-4b02-9cba-08e377b14063","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6816","nonce":1013115018},{"type":"v1","addr":"172.21.15.132:6817","nonce":1013115018}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6818","nonce":1013115018},{"type":"v1","addr":"172.21.15.132:6819","nonce":1013115018}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6822","nonce":1013115018},{"type":"v1","addr":"172.21.15.132:6823","nonce":1013115018}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6820","nonce":1013115018},{"type":"v1","addr":"172.21.15.132:6821","nonce":1013115018}]},"public_addr":"172.21.15.132:6817/1013115018","cluster_addr":"172.21.15.132:6819/1013115018","heartbeat_back_addr":"172.21.15.132:6823/1013115018","heartbeat_front_addr":"172.21.15.132:6821/1013115018","state":["exists","up"]},{"osd":7,"uuid":"e3c95f34-9aee-4cd5-809f-9068fb32a376","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6824","nonce":3753810644},{"type":"v1","addr":"172.21.15.132:6825","nonce":3753810644}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6826","nonce":3753810644},{"type":"v1","addr":"172.21.15.132:6827","nonce":3753810644}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6830","nonce":3753810644},{"type":"v1","addr":"172.21.15.132:6831","nonce":3753810644}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6828","nonce":3753810644},{"type":"v1","addr":"172.21.15.132:6829","nonce":3753810644}]},"public_addr":"172.21.15.132:6825/3753810644","cluster_addr":"172.21.15.132:6827/3753810644","heartbeat_back_addr":"172.21.15.132:6831/3753810644","heartbeat_front_addr":"172.21.15.132:6829/3753810644","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:45:09.956224+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:45:34.301481+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:45:57.790210+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:46:21.497210+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:46:43.185935+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:47:06.429857+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:47:31.280857+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:47:56.922326+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/1941182422":"2024-05-10T03:48:14.438334+0000","172.21.15.78:0/3325882269":"2024-05-10T03:48:14.438334+0000","172.21.15.78:6800/1457479635":"2024-05-10T03:48:14.438334+0000","172.21.15.78:0/3439374234":"2024-05-10T03:43:52.360677+0000","172.21.15.78:6801/987239519":"2024-05-10T03:43:52.360677+0000","172.21.15.78:6800/987239519":"2024-05-10T03:43:52.360677+0000","172.21.15.78:6800/169193258":"2024-05-10T03:43:28.859080+0000","172.21.15.78:6801/1457479635":"2024-05-10T03:48:14.438334+0000","172.21.15.78:6801/169193258":"2024-05-10T03:43:28.859080+0000","172.21.15.78:0/816857241":"2024-05-10T03:48:14.438334+0000","172.21.15.78:0/1125963846":"2024-05-10T03:48:14.438334+0000","172.21.15.78:0/3754415003":"2024-05-10T03:43:52.360677+0000","172.21.15.78:0/906720789":"2024-05-10T03:43:52.360677+0000","172.21.15.78:0/2700265794":"2024-05-10T03:43:28.859080+0000","172.21.15.78:0/147802961":"2024-05-10T03:43:28.859080+0000","172.21.15.78:0/3364333504":"2024-05-10T03:43:28.859080+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-09T03:48:39.754 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:39 smithi132 bash[39162]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-05-09T03:48:39.754 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:39 smithi132 bash[39162]: Writing manifest to image destination 2024-05-09T03:48:40.061 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:39 smithi078 ceph-mon[27629]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:40.061 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:39 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3325896655' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-09T03:48:40.061 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:39 smithi078 ceph-mon[22905]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:40.061 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:39 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3325896655' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-09T03:48:40.085 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-05-09T03:48:40.085 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd dump --format=json 2024-05-09T03:48:40.139 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:39 smithi132 ceph-mon[23807]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:40.139 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:39 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3325896655' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-09T03:48:40.140 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:39 smithi132 podman[39162]: 2024-05-09 03:48:39.745957052 +0000 UTC m=+2.271819511 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-09T03:48:40.140 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:39 smithi132 podman[39162]: 2024-05-09 03:48:39.861419178 +0000 UTC m=+2.387281666 container create dfbd85c54dbd62548e5201ec54d0f9b84cb0a3110587dce8a86b0ed460250c59 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T03:48:40.140 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 podman[39162]: 2024-05-09 03:48:40.03837783 +0000 UTC m=+2.564240279 container init dfbd85c54dbd62548e5201ec54d0f9b84cb0a3110587dce8a86b0ed460250c59 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T03:48:40.140 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 podman[39162]: 2024-05-09 03:48:40.041025963 +0000 UTC m=+2.566888410 container start dfbd85c54dbd62548e5201ec54d0f9b84cb0a3110587dce8a86b0ed460250c59 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T03:48:40.140 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.044Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-09T03:48:40.140 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.044Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-09T03:48:40.140 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=arp 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=edac 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-09T03:48:40.141 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.045Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-09T03:48:40.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=os 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=stat 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=time 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=uname 2024-05-09T03:48:40.143 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-09T03:48:40.144 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-09T03:48:40.144 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-09T03:48:40.144 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-09T03:48:40.144 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[39211]: ts=2024-05-09T03:48:40.046Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-09T03:48:40.144 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 bash[39162]: dfbd85c54dbd62548e5201ec54d0f9b84cb0a3110587dce8a86b0ed460250c59 2024-05-09T03:48:40.309 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:40.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:48:40 smithi132 systemd[1]: Started Ceph node-exporter.b for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:48:41.443 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:41 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:41.443 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:41 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:48:41.443 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:41 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:48:41.443 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:41 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:48:41.443 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:41 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:41.444 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:41 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:48:41.444 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:41 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:48:41.444 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:41 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:48:41.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:41 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:41.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:41 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:48:41.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:41 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:48:41.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:41 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:48:42.380 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:42 smithi078 ceph-mon[22905]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:42.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:42 smithi078 ceph-mon[27629]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:48:42.381 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":59,"fsid":"2777c79e-0db6-11ef-bc97-c7b262605968","created":"2024-05-09T03:42:58.514807+0000","modified":"2024-05-09T03:48:14.438385+0000","last_up_change":"2024-05-09T03:47:57.916295+0000","last_in_change":"2024-05-09T03:47:41.214196+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-09T03:46:00.707353+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cbcf6ecf-5076-4a63-b6f7-df5e34185ddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":3700260173},{"type":"v1","addr":"172.21.15.78:6803","nonce":3700260173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":3700260173},{"type":"v1","addr":"172.21.15.78:6805","nonce":3700260173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":3700260173},{"type":"v1","addr":"172.21.15.78:6809","nonce":3700260173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":3700260173},{"type":"v1","addr":"172.21.15.78:6807","nonce":3700260173}]},"public_addr":"172.21.15.78:6803/3700260173","cluster_addr":"172.21.15.78:6805/3700260173","heartbeat_back_addr":"172.21.15.78:6809/3700260173","heartbeat_front_addr":"172.21.15.78:6807/3700260173","state":["exists","up"]},{"osd":1,"uuid":"25c3b194-8ee3-41d3-a172-90c263918e62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":2683958178},{"type":"v1","addr":"172.21.15.78:6811","nonce":2683958178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":2683958178},{"type":"v1","addr":"172.21.15.78:6813","nonce":2683958178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":2683958178},{"type":"v1","addr":"172.21.15.78:6817","nonce":2683958178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":2683958178},{"type":"v1","addr":"172.21.15.78:6815","nonce":2683958178}]},"public_addr":"172.21.15.78:6811/2683958178","cluster_addr":"172.21.15.78:6813/2683958178","heartbeat_back_addr":"172.21.15.78:6817/2683958178","heartbeat_front_addr":"172.21.15.78:6815/2683958178","state":["exists","up"]},{"osd":2,"uuid":"86841bbd-3c86-4114-9347-e1b9dbd8358f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":3785529399},{"type":"v1","addr":"172.21.15.78:6819","nonce":3785529399}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":3785529399},{"type":"v1","addr":"172.21.15.78:6821","nonce":3785529399}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":3785529399},{"type":"v1","addr":"172.21.15.78:6825","nonce":3785529399}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":3785529399},{"type":"v1","addr":"172.21.15.78:6823","nonce":3785529399}]},"public_addr":"172.21.15.78:6819/3785529399","cluster_addr":"172.21.15.78:6821/3785529399","heartbeat_back_addr":"172.21.15.78:6825/3785529399","heartbeat_front_addr":"172.21.15.78:6823/3785529399","state":["exists","up"]},{"osd":3,"uuid":"97f610ec-76cf-4cb4-8a9a-f111e8b437c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":3522076329},{"type":"v1","addr":"172.21.15.78:6827","nonce":3522076329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":3522076329},{"type":"v1","addr":"172.21.15.78:6829","nonce":3522076329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":3522076329},{"type":"v1","addr":"172.21.15.78:6833","nonce":3522076329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":3522076329},{"type":"v1","addr":"172.21.15.78:6831","nonce":3522076329}]},"public_addr":"172.21.15.78:6827/3522076329","cluster_addr":"172.21.15.78:6829/3522076329","heartbeat_back_addr":"172.21.15.78:6833/3522076329","heartbeat_front_addr":"172.21.15.78:6831/3522076329","state":["exists","up"]},{"osd":4,"uuid":"bd1bd685-528f-41bd-97fc-5aabda193ce2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6800","nonce":268288079},{"type":"v1","addr":"172.21.15.132:6801","nonce":268288079}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6802","nonce":268288079},{"type":"v1","addr":"172.21.15.132:6803","nonce":268288079}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6806","nonce":268288079},{"type":"v1","addr":"172.21.15.132:6807","nonce":268288079}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6804","nonce":268288079},{"type":"v1","addr":"172.21.15.132:6805","nonce":268288079}]},"public_addr":"172.21.15.132:6801/268288079","cluster_addr":"172.21.15.132:6803/268288079","heartbeat_back_addr":"172.21.15.132:6807/268288079","heartbeat_front_addr":"172.21.15.132:6805/268288079","state":["exists","up"]},{"osd":5,"uuid":"30240d46-0459-4fde-ac42-1c4a82b483ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6808","nonce":968710714},{"type":"v1","addr":"172.21.15.132:6809","nonce":968710714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6810","nonce":968710714},{"type":"v1","addr":"172.21.15.132:6811","nonce":968710714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6814","nonce":968710714},{"type":"v1","addr":"172.21.15.132:6815","nonce":968710714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6812","nonce":968710714},{"type":"v1","addr":"172.21.15.132:6813","nonce":968710714}]},"public_addr":"172.21.15.132:6809/968710714","cluster_addr":"172.21.15.132:6811/968710714","heartbeat_back_addr":"172.21.15.132:6815/968710714","heartbeat_front_addr":"172.21.15.132:6813/968710714","state":["exists","up"]},{"osd":6,"uuid":"d6b0f898-9036-4b02-9cba-08e377b14063","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6816","nonce":1013115018},{"type":"v1","addr":"172.21.15.132:6817","nonce":1013115018}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6818","nonce":1013115018},{"type":"v1","addr":"172.21.15.132:6819","nonce":1013115018}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6822","nonce":1013115018},{"type":"v1","addr":"172.21.15.132:6823","nonce":1013115018}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6820","nonce":1013115018},{"type":"v1","addr":"172.21.15.132:6821","nonce":1013115018}]},"public_addr":"172.21.15.132:6817/1013115018","cluster_addr":"172.21.15.132:6819/1013115018","heartbeat_back_addr":"172.21.15.132:6823/1013115018","heartbeat_front_addr":"172.21.15.132:6821/1013115018","state":["exists","up"]},{"osd":7,"uuid":"e3c95f34-9aee-4cd5-809f-9068fb32a376","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6824","nonce":3753810644},{"type":"v1","addr":"172.21.15.132:6825","nonce":3753810644}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6826","nonce":3753810644},{"type":"v1","addr":"172.21.15.132:6827","nonce":3753810644}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6830","nonce":3753810644},{"type":"v1","addr":"172.21.15.132:6831","nonce":3753810644}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.132:6828","nonce":3753810644},{"type":"v1","addr":"172.21.15.132:6829","nonce":3753810644}]},"public_addr":"172.21.15.132:6825/3753810644","cluster_addr":"172.21.15.132:6827/3753810644","heartbeat_back_addr":"172.21.15.132:6831/3753810644","heartbeat_front_addr":"172.21.15.132:6829/3753810644","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:45:09.956224+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:45:34.301481+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:45:57.790210+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:46:21.497210+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:46:43.185935+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:47:06.429857+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:47:31.280857+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-09T03:47:56.922326+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/1941182422":"2024-05-10T03:48:14.438334+0000","172.21.15.78:0/3325882269":"2024-05-10T03:48:14.438334+0000","172.21.15.78:6800/1457479635":"2024-05-10T03:48:14.438334+0000","172.21.15.78:0/3439374234":"2024-05-10T03:43:52.360677+0000","172.21.15.78:6801/987239519":"2024-05-10T03:43:52.360677+0000","172.21.15.78:6800/987239519":"2024-05-10T03:43:52.360677+0000","172.21.15.78:6800/169193258":"2024-05-10T03:43:28.859080+0000","172.21.15.78:6801/1457479635":"2024-05-10T03:48:14.438334+0000","172.21.15.78:6801/169193258":"2024-05-10T03:43:28.859080+0000","172.21.15.78:0/816857241":"2024-05-10T03:48:14.438334+0000","172.21.15.78:0/1125963846":"2024-05-10T03:48:14.438334+0000","172.21.15.78:0/3754415003":"2024-05-10T03:43:52.360677+0000","172.21.15.78:0/906720789":"2024-05-10T03:43:52.360677+0000","172.21.15.78:0/2700265794":"2024-05-10T03:43:28.859080+0000","172.21.15.78:0/147802961":"2024-05-10T03:43:28.859080+0000","172.21.15.78:0/3364333504":"2024-05-10T03:43:28.859080+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-09T03:48:42.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:42 smithi132 ceph-mon[23807]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:43.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:42 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:48:42] "GET /metrics HTTP/1.1" 200 191524 "" "Prometheus/2.33.4" 2024-05-09T03:48:43.152 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-05-09T03:48:43.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-05-09T03:48:43.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-05-09T03:48:43.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-05-09T03:48:43.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-05-09T03:48:43.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-05-09T03:48:43.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-05-09T03:48:43.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-05-09T03:48:43.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:48:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:48:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:43 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3115432179' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-09T03:48:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:43 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:43.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:43 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3115432179' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-09T03:48:43.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:43 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:43.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:43 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3115432179' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-09T03:48:43.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:43 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:43.607 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:43.608 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:43.608 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:43.609 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:43.613 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:44.319 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:44 smithi078 ceph-mon[27629]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:44.319 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:44 smithi078 ceph-mon[22905]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:44 smithi132 ceph-mon[23807]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:45.802 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:45 smithi078 ceph-mon[22905]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:45.802 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:45 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:45.803 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:45 smithi078 ceph-mon[27629]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:45.803 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:45 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:46.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:45 smithi132 ceph-mon[23807]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:46.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:45 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:48:47.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:47 smithi078 ceph-mon[22905]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:47.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:47 smithi078 ceph-mon[27629]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:47.830 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:47.830 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:47.830 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:48.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:47 smithi132 ceph-mon[23807]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:49.364 INFO:teuthology.orchestra.run.smithi078.stdout:34359738414 2024-05-09T03:48:49.364 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-05-09T03:48:50.047 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:49 smithi078 ceph-mon[22905]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:50.048 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:49 smithi078 ceph-mon[27629]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:50.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:49 smithi132 ceph-mon[23807]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:51.930 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:48:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:51 smithi132 ceph-mon[23807]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:52.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:51 smithi078 ceph-mon[22905]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:52.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:51 smithi078 ceph-mon[27629]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:53.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:48:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:48:52] "GET /metrics HTTP/1.1" 200 191512 "" "Prometheus/2.33.4" 2024-05-09T03:48:53.471 INFO:teuthology.orchestra.run.smithi078.stdout:128849018907 2024-05-09T03:48:53.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-05-09T03:48:53.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:48:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:48:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:48:54.220 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:53 smithi078 ceph-mon[22905]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:54.221 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:53 smithi078 ceph-mon[27629]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:54.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:53 smithi132 ceph-mon[23807]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:56.224 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:55 smithi078 ceph-mon[27629]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:56.224 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:55 smithi078 ceph-mon[22905]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:56.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:55 smithi132 ceph-mon[23807]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:56.449 INFO:teuthology.orchestra.run.smithi078.stdout:197568495634 2024-05-09T03:48:56.449 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-05-09T03:48:57.481 INFO:teuthology.orchestra.run.smithi078.stdout:55834574890 2024-05-09T03:48:57.482 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-05-09T03:48:57.902 INFO:teuthology.orchestra.run.smithi078.stdout:107374182432 2024-05-09T03:48:57.902 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-05-09T03:48:58.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:57 smithi132 ceph-mon[23807]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:58.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:57 smithi078 ceph-mon[22905]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:58.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:57 smithi078 ceph-mon[27629]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:48:59.185 INFO:teuthology.orchestra.run.smithi078.stdout:231928233997 2024-05-09T03:48:59.185 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-05-09T03:49:00.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:48:59 smithi132 ceph-mon[23807]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:00.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:48:59 smithi078 ceph-mon[22905]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:00.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:48:59 smithi078 ceph-mon[27629]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:01.982 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:49:01.983 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:49:01.983 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:49:01.983 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:49:02.004 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:49:02.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:01 smithi132 ceph-mon[23807]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:02.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:01 smithi078 ceph-mon[22905]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:02.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:01 smithi078 ceph-mon[27629]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:02.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:49:02 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:49:02] "GET /metrics HTTP/1.1" 200 191499 "" "Prometheus/2.33.4" 2024-05-09T03:49:02.879 INFO:teuthology.orchestra.run.smithi078.stdout:77309411367 2024-05-09T03:49:02.879 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-05-09T03:49:03.134 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:03 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:03.134 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:03 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:03.134 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:03 smithi078 ceph-mon[27629]: Deploying daemon alertmanager.a on smithi078 2024-05-09T03:49:03.134 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:03 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:03.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:03 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:03.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:03 smithi078 ceph-mon[22905]: Deploying daemon alertmanager.a on smithi078 2024-05-09T03:49:03.382 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:03 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:03.382 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:03 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:03.382 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:03 smithi132 ceph-mon[23807]: Deploying daemon alertmanager.a on smithi078 2024-05-09T03:49:03.542 INFO:teuthology.orchestra.run.smithi078.stdout:34359738417 2024-05-09T03:49:03.753 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:49:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:49:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:49:04.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:04 smithi078 ceph-mon[22905]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:04.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:04 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2295037903' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-09T03:49:04.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:04 smithi078 ceph-mon[27629]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:04.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:04 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2295037903' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-09T03:49:04.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:04 smithi132 ceph-mon[23807]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:04.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:04 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2295037903' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-09T03:49:04.652 INFO:teuthology.orchestra.run.smithi078.stdout:163208757273 2024-05-09T03:49:04.653 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-05-09T03:49:06.361 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:49:06.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:06 smithi132 ceph-mon[23807]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:06.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:06 smithi078 ceph-mon[22905]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:06.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:06 smithi078 ceph-mon[27629]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:07.845 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:49:08.330 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:08 smithi078 ceph-mon[27629]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:08.330 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:08 smithi078 ceph-mon[22905]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:08.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:08 smithi132 ceph-mon[23807]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:10.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:10 smithi078 ceph-mon[22905]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:10.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:10 smithi078 ceph-mon[27629]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:10.308 INFO:teuthology.orchestra.run.smithi078.stdout:107374182435 2024-05-09T03:49:10.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:10 smithi132 ceph-mon[23807]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:11.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:11 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1038922418' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-09T03:49:11.475 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:11 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1038922418' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-09T03:49:11.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:11 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1038922418' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-09T03:49:11.931 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182432 got 107374182435 for osd.3 2024-05-09T03:49:11.931 DEBUG:teuthology.parallel:result is None 2024-05-09T03:49:12.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:12 smithi078 ceph-mon[27629]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:12 smithi078 ceph-mon[22905]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:12 smithi132 ceph-mon[23807]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:12.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:49:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:49:12] "GET /metrics HTTP/1.1" 200 191499 "" "Prometheus/2.33.4" 2024-05-09T03:49:13.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:49:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:49:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:49:14.444 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:14 smithi078 ceph-mon[27629]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:14.444 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:14 smithi078 ceph-mon[22905]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:14.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:14 smithi132 ceph-mon[23807]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:15.405 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:49:15.405 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:49:15.405 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:49:15.405 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:49:15.406 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:49:15.406 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:49:15.406 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:49:15.406 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:49:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:49:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:49:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:49:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:49:15.752 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738417 for osd.0 2024-05-09T03:49:15.752 DEBUG:teuthology.parallel:result is None 2024-05-09T03:49:16.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:16 smithi078 ceph-mon[22905]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:16.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:16 smithi078 ceph-mon[27629]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:16 smithi132 ceph-mon[23807]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:16.872 INFO:teuthology.orchestra.run.smithi078.stdout:55834574894 2024-05-09T03:49:16.940 INFO:teuthology.orchestra.run.smithi078.stdout:231928234000 2024-05-09T03:49:17.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:17 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/48351959' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-09T03:49:17.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:17 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/4229292069' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-09T03:49:17.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:17 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/48351959' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-09T03:49:17.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:17 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/4229292069' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-09T03:49:17.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:17 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/48351959' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-09T03:49:17.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:17 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/4229292069' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-09T03:49:17.701 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233997 got 231928234000 for osd.7 2024-05-09T03:49:17.701 DEBUG:teuthology.parallel:result is None 2024-05-09T03:49:18.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:18 smithi078 ceph-mon[22905]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:18.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:18 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2040574926' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-09T03:49:18.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:18 smithi078 ceph-mon[27629]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:18.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:18 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2040574926' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-09T03:49:18.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:18 smithi132 ceph-mon[23807]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:18.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:18 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2040574926' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-09T03:49:18.516 INFO:teuthology.orchestra.run.smithi078.stdout:128849018913 2024-05-09T03:49:19.035 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018907 got 128849018913 for osd.4 2024-05-09T03:49:19.036 DEBUG:teuthology.parallel:result is None 2024-05-09T03:49:19.536 INFO:teuthology.orchestra.run.smithi078.stdout:77309411370 2024-05-09T03:49:20.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:20 smithi132 ceph-mon[23807]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:20.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:20 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2460007359' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-09T03:49:20.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:20 smithi078 ceph-mon[22905]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:20.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:20 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2460007359' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-09T03:49:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:20 smithi078 ceph-mon[27629]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:20 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2460007359' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-09T03:49:21.045 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411367 got 77309411370 for osd.2 2024-05-09T03:49:21.045 DEBUG:teuthology.parallel:result is None 2024-05-09T03:49:21.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:21 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3927938453' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-09T03:49:21.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:21 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3927938453' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-09T03:49:21.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:21 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3927938453' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-09T03:49:21.777 INFO:teuthology.orchestra.run.smithi078.stdout:197568495639 2024-05-09T03:49:22.144 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:22 smithi078 ceph-mon[22905]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:22.144 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:22 smithi078 ceph-mon[27629]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:22.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:22 smithi132 ceph-mon[23807]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:22.515 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495634 got 197568495639 for osd.6 2024-05-09T03:49:22.515 DEBUG:teuthology.parallel:result is None 2024-05-09T03:49:22.762 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:49:22 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:49:22] "GET /metrics HTTP/1.1" 200 191496 "" "Prometheus/2.33.4" 2024-05-09T03:49:22.925 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574894 for osd.1 2024-05-09T03:49:22.925 DEBUG:teuthology.parallel:result is None 2024-05-09T03:49:23.301 INFO:teuthology.orchestra.run.smithi078.stdout:163208757277 2024-05-09T03:49:23.643 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:23 smithi078 podman[45571]: 2024-05-09 03:49:23.470962751 +0000 UTC m=+0.235687591 container create d1a5546ccafc09c24e60a31332a45ba24fc278718594628269df022c45cb07f0 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:23.753 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:49:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:49:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:49:24.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:24 smithi078 ceph-mon[27629]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:24.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:24 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3367777439' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-09T03:49:24.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:24 smithi078 ceph-mon[22905]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:24.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:24 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3367777439' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-09T03:49:24.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:23 smithi078 podman[45571]: 2024-05-09 03:49:23.932803552 +0000 UTC m=+0.697528358 container init d1a5546ccafc09c24e60a31332a45ba24fc278718594628269df022c45cb07f0 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:24.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:23 smithi078 podman[45571]: 2024-05-09 03:49:23.935793592 +0000 UTC m=+0.700518391 container start d1a5546ccafc09c24e60a31332a45ba24fc278718594628269df022c45cb07f0 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:24.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[45599]: level=info ts=2024-05-09T03:49:23.946Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-05-09T03:49:24.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[45599]: level=info ts=2024-05-09T03:49:23.946Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-05-09T03:49:24.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[45599]: level=info ts=2024-05-09T03:49:23.947Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.78 port=9094 2024-05-09T03:49:24.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[45599]: level=info ts=2024-05-09T03:49:23.948Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-09T03:49:24.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[45599]: level=info ts=2024-05-09T03:49:23.973Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-09T03:49:24.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[45599]: level=info ts=2024-05-09T03:49:23.974Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-09T03:49:24.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[45599]: level=info ts=2024-05-09T03:49:23.976Z caller=main.go:518 msg=Listening address=:9093 2024-05-09T03:49:24.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[45599]: level=info ts=2024-05-09T03:49:23.976Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-05-09T03:49:24.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:24 smithi078 bash[45571]: d1a5546ccafc09c24e60a31332a45ba24fc278718594628269df022c45cb07f0 2024-05-09T03:49:24.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:24 smithi078 systemd[1]: Started Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:49:24.484 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757273 got 163208757277 for osd.5 2024-05-09T03:49:24.484 DEBUG:teuthology.parallel:result is None 2024-05-09T03:49:24.485 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-05-09T03:49:24.485 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph pg dump --format=json 2024-05-09T03:49:24.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:24 smithi132 ceph-mon[23807]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:24.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:24 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3367777439' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-09T03:49:24.702 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:49:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:25 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:25 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:25 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:25 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-09T03:49:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:25 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-09T03:49:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:25 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:25 smithi132 ceph-mon[23807]: Deploying daemon grafana.a on smithi132 2024-05-09T03:49:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:25 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-09T03:49:25.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-09T03:49:25.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[27629]: Deploying daemon grafana.a on smithi132 2024-05-09T03:49:25.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-09T03:49:25.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-09T03:49:25.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:25.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[22905]: Deploying daemon grafana.a on smithi132 2024-05-09T03:49:25.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:25 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:26.038 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:25 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[45599]: level=info ts=2024-05-09T03:49:25.949Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000412038s 2024-05-09T03:49:26.420 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:49:26.421 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:26 smithi078 ceph-mon[22905]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:26.421 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:26 smithi078 ceph-mon[27629]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:26.422 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-05-09T03:49:26.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:26 smithi132 ceph-mon[23807]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:27.041 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-05-09T03:49:24.514339+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48848,"kb_used_data":6096,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912368,"statfs":{"total":767960285184,"available":767910264832,"internally_reserved":0,"allocated":6242304,"data_stored":3488866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002325"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":266,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-05-09T03:48:14.578958+0000","last_change":"2024-05-09T03:48:02.029182+0000","last_active":"2024-05-09T03:48:14.578958+0000","last_peered":"2024-05-09T03:48:14.578958+0000","last_clean":"2024-05-09T03:48:14.578958+0000","last_became_active":"2024-05-09T03:48:02.028771+0000","last_became_peered":"2024-05-09T03:48:02.028771+0000","last_unstale":"2024-05-09T03:48:14.578958+0000","last_undegraded":"2024-05-09T03:48:14.578958+0000","last_fullsized":"2024-05-09T03:48:14.578958+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-09T03:46:00.712632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-09T03:46:00.712632+0000","last_clean_scrub_stamp":"2024-05-09T03:46:00.712632+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-10T13:27:31.198838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234002,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6084,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739068,"statfs":{"total":95995035648,"available":95988805632,"internally_reserved":0,"allocated":1110016,"data_stored":766446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.048,"5min":1.048,"15min":1.048},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.5}]},{"osd":1,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.56399999999999995}]},{"osd":2,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.57699999999999996}]},{"osd":3,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.59699999999999998}]},{"osd":4,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.43099999999999999}]},{"osd":5,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.45500000000000002}]},{"osd":6,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.55000000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495640,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.67100000000000004}]},{"osd":1,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.627},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.63800000000000001}]},{"osd":2,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.61899999999999999}]},{"osd":3,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.71999999999999997}]},{"osd":4,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.48099999999999998}]},{"osd":5,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.53300000000000003}]},{"osd":7,"last update":"Thu May 9 03:48:59 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.68300000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574896,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6336,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738816,"statfs":{"total":95995035648,"available":95988547584,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:48:42 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34899999999999998,"15min":0.33500000000000002},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.5,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.374,"5min":0.34999999999999998,"15min":0.33600000000000002},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.48799999999999999,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.47699999999999998}]},{"osd":2,"last update":"Thu May 9 03:49:02 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.26700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.501},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.377,"15min":0.371},"min":{"1min":0.26000000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.59399999999999997}]},{"osd":3,"last update":"Thu May 9 03:48:29 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.437,"15min":0.436},"min":{"1min":0.33500000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42499999999999999,"15min":0.42599999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.58099999999999996}]},{"osd":4,"last update":"Thu May 9 03:48:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.59199999999999997,"15min":0.59399999999999997},"min":{"1min":0.41599999999999998,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.90200000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.59599999999999997,"15min":0.59799999999999998},"min":{"1min":0.434,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.82399999999999995}]},{"osd":5,"last update":"Thu May 9 03:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.58599999999999997,"15min":0.57999999999999996},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.58599999999999997,"15min":0.58299999999999996},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.74199999999999999}]},{"osd":6,"last update":"Thu May 9 03:48:34 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.92800000000000005},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.81599999999999995}]},{"osd":7,"last update":"Thu May 9 03:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.94599999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6920,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738232,"statfs":{"total":95995035648,"available":95987949568,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu May 9 03:48:37 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.34699999999999998,"15min":0.33800000000000002},"min":{"1min":0.26200000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.47699999999999998,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.36799999999999999,"15min":0.35499999999999998},"min":{"1min":0.23799999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.56499999999999995,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.50700000000000001}]},{"osd":2,"last update":"Thu May 9 03:49:05 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.39400000000000002,"15min":0.39600000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.51400000000000001,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.33600000000000002,"5min":0.374,"15min":0.378},"min":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.501,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.51800000000000002}]},{"osd":3,"last update":"Thu May 9 03:48:24 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.309,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.64200000000000002,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.311,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.59399999999999997,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.53000000000000003}]},{"osd":4,"last update":"Thu May 9 03:48:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.60199999999999998,"15min":0.60399999999999998},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.625,"15min":0.626},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.749}]},{"osd":5,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.59799999999999998,"15min":0.60799999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.71899999999999997,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.621,"15min":0.627},"min":{"1min":0.46700000000000003,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.79700000000000004,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.79300000000000004}]},{"osd":6,"last update":"Thu May 9 03:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.83699999999999997},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.80200000000000005}]},{"osd":7,"last update":"Thu May 9 03:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.76600000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6272,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738880,"statfs":{"total":95995035648,"available":95988613120,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:49:07 2024","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.34200000000000003,"15min":0.33000000000000002},"min":{"1min":0.245,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.51300000000000001,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.373,"5min":0.36799999999999999,"15min":0.35199999999999998},"min":{"1min":0.23699999999999999,"5min":0.221,"15min":0.221},"max":{"1min":0.55900000000000005,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.26900000000000002}]},{"osd":1,"last update":"Thu May 9 03:49:07 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.36899999999999999,"15min":0.35499999999999998},"min":{"1min":0.248,"5min":0.217,"15min":0.217},"max":{"1min":0.54200000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.377,"15min":0.35899999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.57399999999999995,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.316}]},{"osd":3,"last update":"Thu May 9 03:48:26 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.45800000000000002,"15min":0.45600000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.59799999999999998,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.33300000000000002}]},{"osd":4,"last update":"Thu May 9 03:48:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.57699999999999996,"15min":0.58099999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.74299999999999999,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60899999999999999,"15min":0.60999999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.80400000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.377}]},{"osd":5,"last update":"Thu May 9 03:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.59999999999999998,"15min":0.60199999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.82199999999999995,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.627,"15min":0.629},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83199999999999996,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.56599999999999995}]},{"osd":6,"last update":"Thu May 9 03:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.48799999999999999}]},{"osd":7,"last update":"Thu May 9 03:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.503},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.47799999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182438,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:48:26 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.379,"15min":0.38300000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.47899999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.378,"5min":0.40200000000000002,"15min":0.40600000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.59899999999999998,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.30599999999999999}]},{"osd":1,"last update":"Thu May 9 03:48:26 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.39300000000000002,"15min":0.39800000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.501,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.40600000000000003,"15min":0.40799999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.29299999999999998}]},{"osd":2,"last update":"Thu May 9 03:48:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.437,"15min":0.441},"min":{"1min":0.32200000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.51800000000000002,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.433,"15min":0.437},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.58399999999999996,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.27500000000000002}]},{"osd":4,"last update":"Thu May 9 03:48:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.60299999999999998,"15min":0.59699999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.137,"5min":3.137,"15min":3.137},"last":0.505},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.59399999999999997,"15min":0.59999999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80300000000000005,"5min":0.873,"15min":0.873},"last":0.66400000000000003}]},{"osd":5,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.58799999999999997,"15min":0.59299999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76700000000000002,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.60199999999999998,"15min":0.60799999999999998},"min":{"1min":0.40799999999999997,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.74099999999999999,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.54800000000000004}]},{"osd":6,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.63100000000000001}]},{"osd":7,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.63800000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018914,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:48:47 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.56200000000000006,"15min":0.57199999999999995},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.58499999999999996,"15min":0.59599999999999997},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.71699999999999997,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.77600000000000002}]},{"osd":1,"last update":"Thu May 9 03:48:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.56499999999999995,"15min":0.57199999999999995},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.72699999999999998,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.498,"5min":0.58599999999999997,"15min":0.60099999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65400000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.70899999999999996}]},{"osd":2,"last update":"Thu May 9 03:48:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.54800000000000004,"15min":0.55500000000000005},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.59599999999999997,"15min":0.60999999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.71199999999999997,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.72999999999999998}]},{"osd":3,"last update":"Thu May 9 03:48:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.59799999999999998,"15min":0.60599999999999998},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.56799999999999995,"15min":0.57799999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.68999999999999995,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.65100000000000002}]},{"osd":5,"last update":"Thu May 9 03:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.307,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.60899999999999999,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.28599999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56799999999999995,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.63300000000000001}]},{"osd":6,"last update":"Thu May 9 03:48:34 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.60799999999999998}]},{"osd":7,"last update":"Thu May 9 03:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.59499999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757277,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5628,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739524,"statfs":{"total":95995035648,"available":95989272576,"internally_reserved":0,"allocated":577536,"data_stored":237518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.51700000000000002,"15min":0.51200000000000001},"min":{"1min":0.372,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.50800000000000001,"15min":0.505},"min":{"1min":0.42699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.51300000000000001}]},{"osd":1,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.50600000000000001,"15min":0.503},"min":{"1min":0.41899999999999998,"5min":0.317,"15min":0.317},"max":{"1min":0.64400000000000002,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54700000000000004,"15min":0.54900000000000004},"min":{"1min":0.36399999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.64400000000000002,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.626}]},{"osd":2,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.38200000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.53400000000000003,"15min":0.53100000000000003},"min":{"1min":0.38900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69499999999999995,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.52200000000000002}]},{"osd":3,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52100000000000002,"15min":0.51900000000000002},"min":{"1min":0.38500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.54400000000000004,"15min":0.54100000000000004},"min":{"1min":0.45100000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.60899999999999999}]},{"osd":4,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.40500000000000003,"15min":0.39300000000000002},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.41999999999999998,"15min":0.41199999999999998},"min":{"1min":0.33300000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.67500000000000004,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.496}]},{"osd":6,"last update":"Thu May 9 03:48:38 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.58199999999999996}]},{"osd":7,"last update":"Thu May 9 03:49:02 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.67800000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-09T03:49:27.046 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph pg dump --format=json 2024-05-09T03:49:27.260 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:49:27.503 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:27 smithi078 ceph-mon[27629]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-09T03:49:27.503 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:27 smithi078 ceph-mon[22905]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-09T03:49:27.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:27 smithi132 ceph-mon[23807]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-09T03:49:28.347 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:28 smithi078 ceph-mon[22905]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:28.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:28 smithi078 ceph-mon[27629]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:28 smithi132 ceph-mon[23807]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:29.001 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:49:29.002 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-05-09T03:49:29.617 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":40,"stamp":"2024-05-09T03:49:28.515050+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48848,"kb_used_data":6096,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912368,"statfs":{"total":767960285184,"available":767910264832,"internally_reserved":0,"allocated":6242304,"data_stored":3488866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002345"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":266,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-05-09T03:48:14.578958+0000","last_change":"2024-05-09T03:48:02.029182+0000","last_active":"2024-05-09T03:48:14.578958+0000","last_peered":"2024-05-09T03:48:14.578958+0000","last_clean":"2024-05-09T03:48:14.578958+0000","last_became_active":"2024-05-09T03:48:02.028771+0000","last_became_peered":"2024-05-09T03:48:02.028771+0000","last_unstale":"2024-05-09T03:48:14.578958+0000","last_undegraded":"2024-05-09T03:48:14.578958+0000","last_fullsized":"2024-05-09T03:48:14.578958+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-09T03:46:00.712632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-09T03:46:00.712632+0000","last_clean_scrub_stamp":"2024-05-09T03:46:00.712632+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-10T13:27:31.198838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234003,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6084,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739068,"statfs":{"total":95995035648,"available":95988805632,"internally_reserved":0,"allocated":1110016,"data_stored":766446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.048,"5min":1.048,"15min":1.048},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.42799999999999999}]},{"osd":1,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.436},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.443}]},{"osd":2,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.40000000000000002}]},{"osd":3,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.48999999999999999}]},{"osd":4,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.311},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.29599999999999999}]},{"osd":5,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.40899999999999997}]},{"osd":6,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.377},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.34200000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495641,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.53000000000000003}]},{"osd":1,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.51100000000000001}]},{"osd":2,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.56299999999999994}]},{"osd":3,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.49299999999999999}]},{"osd":4,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.46200000000000002}]},{"osd":5,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.47799999999999998}]},{"osd":7,"last update":"Thu May 9 03:48:59 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.501},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.45100000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6336,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738816,"statfs":{"total":95995035648,"available":95988547584,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:48:42 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34899999999999998,"15min":0.33500000000000002},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.5,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.374,"5min":0.34999999999999998,"15min":0.33600000000000002},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.48799999999999999,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.40600000000000003}]},{"osd":2,"last update":"Thu May 9 03:49:02 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.26700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.377,"15min":0.371},"min":{"1min":0.26000000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.433}]},{"osd":3,"last update":"Thu May 9 03:48:29 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.437,"15min":0.436},"min":{"1min":0.33500000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42499999999999999,"15min":0.42599999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.45500000000000002}]},{"osd":4,"last update":"Thu May 9 03:48:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.59199999999999997,"15min":0.59399999999999997},"min":{"1min":0.41599999999999998,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.502},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.59599999999999997,"15min":0.59799999999999998},"min":{"1min":0.434,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.58999999999999997}]},{"osd":5,"last update":"Thu May 9 03:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.58599999999999997,"15min":0.57999999999999996},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.498},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.58599999999999997,"15min":0.58299999999999996},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.60499999999999998}]},{"osd":6,"last update":"Thu May 9 03:48:34 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.61599999999999999}]},{"osd":7,"last update":"Thu May 9 03:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.623},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.59799999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6920,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738232,"statfs":{"total":95995035648,"available":95987949568,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu May 9 03:48:37 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.34699999999999998,"15min":0.33800000000000002},"min":{"1min":0.26200000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.47699999999999998,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.36799999999999999,"15min":0.35499999999999998},"min":{"1min":0.23799999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.56499999999999995,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.20200000000000001}]},{"osd":2,"last update":"Thu May 9 03:49:05 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.39400000000000002,"15min":0.39600000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.51400000000000001,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.33600000000000002,"5min":0.374,"15min":0.378},"min":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.501,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.28999999999999998}]},{"osd":3,"last update":"Thu May 9 03:48:24 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.309,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.64200000000000002,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.311,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.59399999999999997,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.27500000000000002}]},{"osd":4,"last update":"Thu May 9 03:48:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.60199999999999998,"15min":0.60399999999999998},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.62,"5min":0.625,"15min":0.626},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.622}]},{"osd":5,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.59799999999999998,"15min":0.60799999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.71899999999999997,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.629},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.621,"15min":0.627},"min":{"1min":0.46700000000000003,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.79700000000000004,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.57799999999999996}]},{"osd":6,"last update":"Thu May 9 03:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.58499999999999996}]},{"osd":7,"last update":"Thu May 9 03:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.66400000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6272,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738880,"statfs":{"total":95995035648,"available":95988613120,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:49:07 2024","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.34200000000000003,"15min":0.33000000000000002},"min":{"1min":0.245,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.51300000000000001,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.373,"5min":0.36799999999999999,"15min":0.35199999999999998},"min":{"1min":0.23699999999999999,"5min":0.221,"15min":0.221},"max":{"1min":0.55900000000000005,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.39100000000000001}]},{"osd":1,"last update":"Thu May 9 03:49:07 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.36899999999999999,"15min":0.35499999999999998},"min":{"1min":0.248,"5min":0.217,"15min":0.217},"max":{"1min":0.54200000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.377,"15min":0.35899999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.57399999999999995,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.40500000000000003}]},{"osd":3,"last update":"Thu May 9 03:48:26 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.45800000000000002,"15min":0.45600000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.59799999999999998,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.35299999999999998}]},{"osd":4,"last update":"Thu May 9 03:48:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.57699999999999996,"15min":0.58099999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.74299999999999999,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60899999999999999,"15min":0.60999999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.80400000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.754}]},{"osd":5,"last update":"Thu May 9 03:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.59999999999999998,"15min":0.60199999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.82199999999999995,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.627,"15min":0.629},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83199999999999996,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.77200000000000002}]},{"osd":6,"last update":"Thu May 9 03:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.78000000000000003}]},{"osd":7,"last update":"Thu May 9 03:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.71599999999999997}]}]},{"osd":3,"up_from":25,"seq":107374182439,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:48:26 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.379,"15min":0.38300000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.47899999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.378,"5min":0.40200000000000002,"15min":0.40600000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.59899999999999998,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.54500000000000004}]},{"osd":1,"last update":"Thu May 9 03:48:26 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.39300000000000002,"15min":0.39800000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.501,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.40600000000000003,"15min":0.40799999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.53300000000000003}]},{"osd":2,"last update":"Thu May 9 03:48:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.437,"15min":0.441},"min":{"1min":0.32200000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.51800000000000002,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.496},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.433,"15min":0.437},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.58399999999999996,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.51600000000000001}]},{"osd":4,"last update":"Thu May 9 03:48:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.60299999999999998,"15min":0.59699999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.137,"5min":3.137,"15min":3.137},"last":0.627},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.59399999999999997,"15min":0.59999999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80300000000000005,"5min":0.873,"15min":0.873},"last":0.69699999999999995}]},{"osd":5,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.58799999999999997,"15min":0.59299999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76700000000000002,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.60199999999999998,"15min":0.60799999999999998},"min":{"1min":0.40799999999999997,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.74099999999999999,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.65900000000000003}]},{"osd":6,"last update":"Thu May 9 03:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.64800000000000002}]},{"osd":7,"last update":"Thu May 9 03:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.72099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018915,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:48:47 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.56200000000000006,"15min":0.57199999999999995},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.58499999999999996,"15min":0.59599999999999997},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.71699999999999997,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.73999999999999999}]},{"osd":1,"last update":"Thu May 9 03:48:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.56499999999999995,"15min":0.57199999999999995},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.72699999999999998,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.498,"5min":0.58599999999999997,"15min":0.60099999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65400000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.60799999999999998}]},{"osd":2,"last update":"Thu May 9 03:48:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.54800000000000004,"15min":0.55500000000000005},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.59599999999999997,"15min":0.60999999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.71199999999999997,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.70999999999999996}]},{"osd":3,"last update":"Thu May 9 03:48:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.59799999999999998,"15min":0.60599999999999998},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.56799999999999995,"15min":0.57799999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.68999999999999995,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.499}]},{"osd":5,"last update":"Thu May 9 03:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.307,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.60899999999999999,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.63},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.28599999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56799999999999995,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.54100000000000004}]},{"osd":6,"last update":"Thu May 9 03:48:34 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.64600000000000002}]},{"osd":7,"last update":"Thu May 9 03:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.70099999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757278,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5628,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739524,"statfs":{"total":95995035648,"available":95989272576,"internally_reserved":0,"allocated":577536,"data_stored":237518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.51700000000000002,"15min":0.51200000000000001},"min":{"1min":0.372,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.50800000000000001,"15min":0.505},"min":{"1min":0.42699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.44800000000000001}]},{"osd":1,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.50600000000000001,"15min":0.503},"min":{"1min":0.41899999999999998,"5min":0.317,"15min":0.317},"max":{"1min":0.64400000000000002,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54700000000000004,"15min":0.54900000000000004},"min":{"1min":0.36399999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.64400000000000002,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.49099999999999999}]},{"osd":2,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.38200000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.53400000000000003,"15min":0.53100000000000003},"min":{"1min":0.38900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69499999999999995,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.53000000000000003}]},{"osd":3,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52100000000000002,"15min":0.51900000000000002},"min":{"1min":0.38500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.54400000000000004,"15min":0.54100000000000004},"min":{"1min":0.45100000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.39300000000000002}]},{"osd":4,"last update":"Thu May 9 03:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.40500000000000003,"15min":0.39300000000000002},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.41999999999999998,"15min":0.41199999999999998},"min":{"1min":0.33300000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.67500000000000004,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.435}]},{"osd":6,"last update":"Thu May 9 03:48:38 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.57599999999999996}]},{"osd":7,"last update":"Thu May 9 03:49:02 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.622}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-09T03:49:29.618 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-05-09T03:49:29.618 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-05-09T03:49:29.619 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-05-09T03:49:29.619 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- ceph health --format=json 2024-05-09T03:49:29.831 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/mon.a/config 2024-05-09T03:49:30.493 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:30 smithi078 ceph-mon[22905]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:30.493 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:30 smithi078 ceph-mon[22905]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-09T03:49:30.493 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:30 smithi078 ceph-mon[27629]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:30.493 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:30 smithi078 ceph-mon[27629]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-09T03:49:30.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:30 smithi132 ceph-mon[23807]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:30.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:30 smithi132 ceph-mon[23807]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-09T03:49:31.450 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:49:31.451 INFO:teuthology.orchestra.run.smithi078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-05-09T03:49:32.068 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-05-09T03:49:32.068 INFO:tasks.cephadm:Setup complete, yielding 2024-05-09T03:49:32.068 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-09T03:49:32.077 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi078.front.sepia.ceph.com 2024-05-09T03:49:32.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-05-09T03:49:32.381 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:32 smithi078 ceph-mon[22905]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:32.381 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:32 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1611197253' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-09T03:49:32.381 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:32 smithi078 ceph-mon[27629]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:32.381 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:32 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1611197253' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-09T03:49:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:32 smithi132 ceph-mon[23807]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:32 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1611197253' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-09T03:49:33.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:49:32 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:49:32] "GET /metrics HTTP/1.1" 200 191501 "" "Prometheus/2.33.4" 2024-05-09T03:49:33.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:49:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:49:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:49:34.247 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[45599]: level=info ts=2024-05-09T03:49:33.951Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002461936s 2024-05-09T03:49:34.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:34 smithi132 ceph-mon[23807]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:34.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:34 smithi078 ceph-mon[22905]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:34.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:34 smithi078 ceph-mon[27629]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:35.279 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T03:49:35.280 INFO:teuthology.orchestra.run.smithi078.stdout: "id": "0ae48363-ad56-463d-95e8-e141d876bdce", 2024-05-09T03:49:35.280 INFO:teuthology.orchestra.run.smithi078.stdout: "name": "r", 2024-05-09T03:49:35.280 INFO:teuthology.orchestra.run.smithi078.stdout: "current_period": "6dadd3f6-4934-4c46-821d-a410031d04a4", 2024-05-09T03:49:35.280 INFO:teuthology.orchestra.run.smithi078.stdout: "epoch": 1 2024-05-09T03:49:35.280 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T03:49:35.541 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:35 smithi132 ceph-mon[23807]: osdmap e60: 8 total, 8 up, 8 in 2024-05-09T03:49:35.541 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:35 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2997517301' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-05-09T03:49:35.541 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:35 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2997517301' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-05-09T03:49:35.541 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:35 smithi132 ceph-mon[23807]: osdmap e61: 8 total, 8 up, 8 in 2024-05-09T03:49:35.621 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:35 smithi078 ceph-mon[22905]: osdmap e60: 8 total, 8 up, 8 in 2024-05-09T03:49:35.622 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:35 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2997517301' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-05-09T03:49:35.622 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:35 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2997517301' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-05-09T03:49:35.622 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:35 smithi078 ceph-mon[22905]: osdmap e61: 8 total, 8 up, 8 in 2024-05-09T03:49:35.622 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:35 smithi078 ceph-mon[27629]: osdmap e60: 8 total, 8 up, 8 in 2024-05-09T03:49:35.622 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:35 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2997517301' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-05-09T03:49:35.622 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:35 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2997517301' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-05-09T03:49:35.623 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:35 smithi078 ceph-mon[27629]: osdmap e61: 8 total, 8 up, 8 in 2024-05-09T03:49:35.865 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-05-09T03:49:36.440 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:36 smithi078 ceph-mon[27629]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:36.440 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:36 smithi078 ceph-mon[27629]: osdmap e62: 8 total, 8 up, 8 in 2024-05-09T03:49:36.441 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:36 smithi078 ceph-mon[22905]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:36.441 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:36 smithi078 ceph-mon[22905]: osdmap e62: 8 total, 8 up, 8 in 2024-05-09T03:49:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:36 smithi132 ceph-mon[23807]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:36 smithi132 ceph-mon[23807]: osdmap e62: 8 total, 8 up, 8 in 2024-05-09T03:49:37.596 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "id": "501e9ebe-1990-4eff-b292-80554f4357fe", 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "name": "default", 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "api_name": "default", 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "is_master": "true", 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "endpoints": [], 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "hostnames": [], 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "hostnames_s3website": [], 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "master_zone": "", 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "zones": [], 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "placement_targets": [], 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "default_placement": "", 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "realm_id": "0ae48363-ad56-463d-95e8-e141d876bdce", 2024-05-09T03:49:37.597 INFO:teuthology.orchestra.run.smithi078.stdout: "sync_policy": { 2024-05-09T03:49:37.598 INFO:teuthology.orchestra.run.smithi078.stdout: "groups": [] 2024-05-09T03:49:37.598 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T03:49:37.598 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T03:49:38.165 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-05-09T03:49:38.190 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 systemd[1]: Starting Ceph grafana.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:49:38.468 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:38 smithi078 ceph-mon[22905]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-09T03:49:38.468 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:38 smithi078 ceph-mon[27629]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-09T03:49:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:38 smithi132 ceph-mon[23807]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-09T03:49:38.504 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 podman[39903]: 2024-05-09 03:49:38.310427091 +0000 UTC m=+0.016680676 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-05-09T03:49:38.504 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 podman[39903]: 2024-05-09 03:49:38.439717762 +0000 UTC m=+0.145971368 container create c1b7ab0860834ba92f21f2cfaa0faf2959435ebe2d6cfaf35a5df9115dd36c2c (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, architecture=x86_64, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, com.redhat.component=ubi8-container, vcs-type=git, name=ubi8, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public) 2024-05-09T03:49:38.813 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 podman[39903]: 2024-05-09 03:49:38.684904363 +0000 UTC m=+0.391157927 container init c1b7ab0860834ba92f21f2cfaa0faf2959435ebe2d6cfaf35a5df9115dd36c2c (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, release=236.1648460182, com.redhat.component=ubi8-container, architecture=x86_64, vendor=Red Hat, Inc., io.buildah.version=1.24.2, distribution-scope=public, io.openshift.expose-services=, vcs-type=git, version=8.5) 2024-05-09T03:49:38.813 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 podman[39903]: 2024-05-09 03:49:38.68818303 +0000 UTC m=+0.394436601 container start c1b7ab0860834ba92f21f2cfaa0faf2959435ebe2d6cfaf35a5df9115dd36c2c (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., version=8.5, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, distribution-scope=public, vcs-type=git, build-date=2022-03-28T10:36:18.413762, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, maintainer=Paul Cuzner , description=Ceph Grafana Container, com.redhat.component=ubi8-container, architecture=x86_64) 2024-05-09T03:49:38.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-05-09T03:49:38.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-05-09T03:49:38.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-05-09T03:49:38.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-05-09T03:49:38.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-05-09T03:49:38.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-05-09T03:49:38.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-05-09T03:49:38.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-05-09T03:49:38.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-05-09T03:49:38.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-05-09T03:49:38.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-05-09T03:49:38.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-05-09T03:49:38.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-05-09T03:49:38.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="App mode production" logger=settings 2024-05-09T03:49:38.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-05-09T03:49:38.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-05-09T03:49:38.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-05-09T03:49:38.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-05-09T03:49:38.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-05-09T03:49:38.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-05-09T03:49:38.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-05-09T03:49:38.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-05-09T03:49:38.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-05-09T03:49:38.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-05-09T03:49:38.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-05-09T03:49:38.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-05-09T03:49:38.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-05-09T03:49:38.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-05-09T03:49:38.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-05-09T03:49:38.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-05-09T03:49:38.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-05-09T03:49:38.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-05-09T03:49:38.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-05-09T03:49:38.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-05-09T03:49:38.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 bash[39903]: c1b7ab0860834ba92f21f2cfaa0faf2959435ebe2d6cfaf35a5df9115dd36c2c 2024-05-09T03:49:38.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-05-09T03:49:38.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 systemd[1]: Started Ceph grafana.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:49:38.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-05-09T03:49:38.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-05-09T03:49:38.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-05-09T03:49:38.819 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-05-09T03:49:38.820 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-05-09T03:49:38.820 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-05-09T03:49:38.820 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-05-09T03:49:38.820 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-05-09T03:49:39.064 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-05-09T03:49:39.064 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-05-09T03:49:39.064 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-05-09T03:49:39.064 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-05-09T03:49:39.065 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-05-09T03:49:39.066 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-05-09T03:49:39.066 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-05-09T03:49:39.066 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-05-09T03:49:39.067 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-05-09T03:49:39.068 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-05-09T03:49:39.068 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-05-09T03:49:39.068 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-05-09T03:49:39.068 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-05-09T03:49:39.068 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-05-09T03:49:39.068 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-05-09T03:49:39.068 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-05-09T03:49:39.068 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-05-09T03:49:39.068 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-05-09T03:49:39.068 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-05-09T03:49:39.068 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-05-09T03:49:39.068 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-05-09T03:49:39.069 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-05-09T03:49:39.070 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-05-09T03:49:39.071 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-05-09T03:49:39.071 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-05-09T03:49:39.071 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-05-09T03:49:39.071 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-05-09T03:49:39.072 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-05-09T03:49:39.072 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-05-09T03:49:39.072 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-05-09T03:49:39.072 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-05-09T03:49:39.072 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-05-09T03:49:39.072 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-05-09T03:49:39.072 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-05-09T03:49:39.072 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-05-09T03:49:39.072 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-05-09T03:49:39.072 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-05-09T03:49:39.072 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-05-09T03:49:39.073 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-05-09T03:49:39.074 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-05-09T03:49:39.075 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-05-09T03:49:39.076 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-05-09T03:49:39.077 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-05-09T03:49:39.078 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-05-09T03:49:39.079 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-05-09T03:49:39.079 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-05-09T03:49:39.079 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-05-09T03:49:39.079 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-05-09T03:49:39.079 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-05-09T03:49:39.079 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-05-09T03:49:39.079 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-05-09T03:49:39.079 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-05-09T03:49:39.079 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-05-09T03:49:39.079 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-05-09T03:49:39.079 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-05-09T03:49:39.079 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-05-09T03:49:39.080 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-05-09T03:49:39.080 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-05-09T03:49:39.080 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-05-09T03:49:39.318 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-05-09T03:49:39.319 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-05-09T03:49:39.319 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-05-09T03:49:39.319 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-05-09T03:49:39.319 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-05-09T03:49:39.319 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-05-09T03:49:39.319 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-05-09T03:49:39.319 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-05-09T03:49:39.319 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-05-09T03:49:39.320 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-05-09T03:49:39.320 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-05-09T03:49:39.320 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-05-09T03:49:39.320 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-05-09T03:49:39.320 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-05-09T03:49:39.320 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-05-09T03:49:39.320 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-05-09T03:49:39.320 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-05-09T03:49:39.320 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-05-09T03:49:39.320 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-05-09T03:49:39.322 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-05-09T03:49:39.322 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-05-09T03:49:39.322 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-05-09T03:49:39.322 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-05-09T03:49:39.322 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-05-09T03:49:39.322 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-05-09T03:49:39.322 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-05-09T03:49:39.322 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-05-09T03:49:39.322 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-05-09T03:49:39.323 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-05-09T03:49:39.323 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-05-09T03:49:39.323 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-05-09T03:49:39.323 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-05-09T03:49:39.323 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-05-09T03:49:39.323 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-05-09T03:49:39.323 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-05-09T03:49:39.323 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-05-09T03:49:39.323 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-05-09T03:49:39.323 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-05-09T03:49:39.323 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-05-09T03:49:39.324 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-05-09T03:49:39.325 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-05-09T03:49:39.325 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-05-09T03:49:39.325 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-05-09T03:49:39.325 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-05-09T03:49:39.325 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-05-09T03:49:39.325 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-05-09T03:49:39.325 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-05-09T03:49:39.325 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-05-09T03:49:39.325 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-05-09T03:49:39.325 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-05-09T03:49:39.325 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-05-09T03:49:39.326 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-05-09T03:49:39.326 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-05-09T03:49:39.326 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-05-09T03:49:39.326 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-05-09T03:49:39.326 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-05-09T03:49:39.326 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-05-09T03:49:39.326 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-05-09T03:49:39.326 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-05-09T03:49:39.326 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-05-09T03:49:39.326 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-05-09T03:49:39.327 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-05-09T03:49:39.327 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-05-09T03:49:39.327 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-05-09T03:49:39.327 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-05-09T03:49:39.327 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-05-09T03:49:39.327 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-05-09T03:49:39.327 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-05-09T03:49:39.327 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-05-09T03:49:39.327 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-05-09T03:49:39.327 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-05-09T03:49:39.327 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-05-09T03:49:39.327 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-05-09T03:49:39.328 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-05-09T03:49:39.328 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-05-09T03:49:39.328 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-05-09T03:49:39.328 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-05-09T03:49:39.328 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-05-09T03:49:39.328 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-05-09T03:49:39.328 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-05-09T03:49:39.328 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-05-09T03:49:39.328 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-05-09T03:49:39.329 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-05-09T03:49:39.329 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-05-09T03:49:39.329 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-05-09T03:49:39.329 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-05-09T03:49:39.329 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-05-09T03:49:39.329 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-05-09T03:49:39.329 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-05-09T03:49:39.329 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-05-09T03:49:39.330 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-05-09T03:49:39.330 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-05-09T03:49:39.330 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-05-09T03:49:39.330 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-05-09T03:49:39.330 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-05-09T03:49:39.330 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-05-09T03:49:39.330 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-05-09T03:49:39.330 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-05-09T03:49:39.330 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-05-09T03:49:39.330 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-05-09T03:49:39.331 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-05-09T03:49:39.332 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-05-09T03:49:39.332 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-05-09T03:49:39.332 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-05-09T03:49:39.332 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-05-09T03:49:39.332 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-05-09T03:49:39.332 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-05-09T03:49:39.332 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=557.305902ms 2024-05-09T03:49:39.332 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-05-09T03:49:39.332 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-05-09T03:49:39.332 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-05-09T03:49:39.332 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-05-09T03:49:39.333 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-05-09T03:49:39.333 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-05-09T03:49:39.584 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-05-09T03:49:39.585 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-05-09T03:49:39.585 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-05-09T03:49:39.585 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-05-09T03:49:39.585 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:49:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:49:39+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-05-09T03:49:40.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:39 smithi132 ceph-mon[23807]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-09T03:49:40.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:39 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:40.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:39 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:49:40.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:39 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:49:40.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:39 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:49:40.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:39 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:40.019 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T03:49:40.019 INFO:teuthology.orchestra.run.smithi078.stdout: "id": "374ba5e6-d9c9-4614-83a5-07436a4990b2", 2024-05-09T03:49:40.019 INFO:teuthology.orchestra.run.smithi078.stdout: "name": "z", 2024-05-09T03:49:40.019 INFO:teuthology.orchestra.run.smithi078.stdout: "domain_root": "z.rgw.meta:root", 2024-05-09T03:49:40.019 INFO:teuthology.orchestra.run.smithi078.stdout: "control_pool": "z.rgw.control", 2024-05-09T03:49:40.019 INFO:teuthology.orchestra.run.smithi078.stdout: "gc_pool": "z.rgw.log:gc", 2024-05-09T03:49:40.019 INFO:teuthology.orchestra.run.smithi078.stdout: "lc_pool": "z.rgw.log:lc", 2024-05-09T03:49:40.019 INFO:teuthology.orchestra.run.smithi078.stdout: "log_pool": "z.rgw.log", 2024-05-09T03:49:40.019 INFO:teuthology.orchestra.run.smithi078.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-05-09T03:49:40.019 INFO:teuthology.orchestra.run.smithi078.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-05-09T03:49:40.019 INFO:teuthology.orchestra.run.smithi078.stdout: "roles_pool": "z.rgw.meta:roles", 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: "otp_pool": "z.rgw.otp", 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: "system_key": { 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: "access_key": "", 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: "secret_key": "" 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: "placement_pools": [ 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-05-09T03:49:40.020 INFO:teuthology.orchestra.run.smithi078.stdout: "key": "default-placement", 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: "val": { 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: "index_pool": "z.rgw.buckets.index", 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: "storage_classes": { 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: "STANDARD": { 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: "data_pool": "z.rgw.buckets.data" 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: "index_type": 0 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: "realm_id": "0ae48363-ad56-463d-95e8-e141d876bdce", 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout: "notif_pool": "z.rgw.log:notif" 2024-05-09T03:49:40.021 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T03:49:40.217 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:39 smithi078 ceph-mon[27629]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-09T03:49:40.217 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:39 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:40.217 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:39 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:49:40.217 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:39 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:49:40.217 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:39 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:49:40.217 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:39 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:40.218 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:39 smithi078 ceph-mon[22905]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-09T03:49:40.218 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:39 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:40.218 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:39 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:49:40.218 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:39 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:49:40.218 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:39 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:49:40.218 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:39 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:40.722 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-05-09T03:49:42.233 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:41 smithi078 ceph-mon[27629]: pgmap v49: 33 pgs: 33 active+clean; 578 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 163 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-05-09T03:49:42.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:41 smithi078 ceph-mon[22905]: pgmap v49: 33 pgs: 33 active+clean; 578 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 163 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-05-09T03:49:42.239 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:41 smithi132 ceph-mon[23807]: pgmap v49: 33 pgs: 33 active+clean; 578 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 163 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-05-09T03:49:42.899 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:49:42 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:49:42] "GET /metrics HTTP/1.1" 200 191501 "" "Prometheus/2.33.4" 2024-05-09T03:49:43.654 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:43 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:43.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:43 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:43.753 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:49:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:49:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:49:43.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:43 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:44.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:44 smithi078 ceph-mon[22905]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-05-09T03:49:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:44 smithi078 ceph-mon[22905]: osdmap e63: 8 total, 8 up, 8 in 2024-05-09T03:49:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:44 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2641723673' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-09T03:49:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:44 smithi078 ceph-mon[27629]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-05-09T03:49:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:44 smithi078 ceph-mon[27629]: osdmap e63: 8 total, 8 up, 8 in 2024-05-09T03:49:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:44 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2641723673' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-09T03:49:44.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:44 smithi132 ceph-mon[23807]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-05-09T03:49:44.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:44 smithi132 ceph-mon[23807]: osdmap e63: 8 total, 8 up, 8 in 2024-05-09T03:49:44.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:44 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2641723673' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-09T03:49:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[22905]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 989 B/s wr, 2 op/s 2024-05-09T03:49:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2641723673' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-05-09T03:49:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[22905]: osdmap e64: 8 total, 8 up, 8 in 2024-05-09T03:49:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[22905]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-09T03:49:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[22905]: Reconfiguring daemon alertmanager.a on smithi078 2024-05-09T03:49:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[27629]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 989 B/s wr, 2 op/s 2024-05-09T03:49:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2641723673' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-05-09T03:49:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[27629]: osdmap e64: 8 total, 8 up, 8 in 2024-05-09T03:49:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[27629]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-09T03:49:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:45 smithi078 ceph-mon[27629]: Reconfiguring daemon alertmanager.a on smithi078 2024-05-09T03:49:46.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:45 smithi132 ceph-mon[23807]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 989 B/s wr, 2 op/s 2024-05-09T03:49:46.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:45 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:46.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:45 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:46.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:45 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2641723673' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-05-09T03:49:46.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:45 smithi132 ceph-mon[23807]: osdmap e64: 8 total, 8 up, 8 in 2024-05-09T03:49:46.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:45 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:46.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:45 smithi132 ceph-mon[23807]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-09T03:49:46.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:45 smithi132 ceph-mon[23807]: Reconfiguring daemon alertmanager.a on smithi078 2024-05-09T03:49:47.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:46 smithi132 ceph-mon[23807]: osdmap e65: 8 total, 8 up, 8 in 2024-05-09T03:49:47.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:46 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2641723673' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-09T03:49:47.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:46 smithi078 ceph-mon[22905]: osdmap e65: 8 total, 8 up, 8 in 2024-05-09T03:49:47.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:46 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2641723673' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-09T03:49:47.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:46 smithi078 ceph-mon[27629]: osdmap e65: 8 total, 8 up, 8 in 2024-05-09T03:49:47.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:46 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2641723673' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-09T03:49:48.216 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:47 smithi078 ceph-mon[27629]: pgmap v55: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1023 B/s wr, 5 op/s 2024-05-09T03:49:48.216 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:47 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2641723673' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-05-09T03:49:48.216 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:47 smithi078 ceph-mon[27629]: osdmap e66: 8 total, 8 up, 8 in 2024-05-09T03:49:48.217 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:47 smithi078 ceph-mon[22905]: pgmap v55: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1023 B/s wr, 5 op/s 2024-05-09T03:49:48.217 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:47 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2641723673' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-05-09T03:49:48.217 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:47 smithi078 ceph-mon[22905]: osdmap e66: 8 total, 8 up, 8 in 2024-05-09T03:49:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:47 smithi132 ceph-mon[23807]: pgmap v55: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1023 B/s wr, 5 op/s 2024-05-09T03:49:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:47 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2641723673' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-05-09T03:49:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:47 smithi132 ceph-mon[23807]: osdmap e66: 8 total, 8 up, 8 in 2024-05-09T03:49:48.512 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:48 smithi078 systemd[1]: Stopping Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:49:48.512 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:48 smithi078 bash[47310]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager.a" found: no such container 2024-05-09T03:49:48.512 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:48 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[45599]: level=info ts=2024-05-09T03:49:48.355Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-05-09T03:49:48.512 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:48 smithi078 podman[47323]: 2024-05-09 03:49:48.376281277 +0000 UTC m=+0.123584972 container died d1a5546ccafc09c24e60a31332a45ba24fc278718594628269df022c45cb07f0 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:48.919 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:48 smithi078 podman[47323]: 2024-05-09 03:49:48.805594552 +0000 UTC m=+0.552898227 container cleanup d1a5546ccafc09c24e60a31332a45ba24fc278718594628269df022c45cb07f0 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:48.919 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:48 smithi078 bash[47323]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a 2024-05-09T03:49:48.919 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:48 smithi078 bash[47352]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager.a" found: no such container 2024-05-09T03:49:49.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:48 smithi132 ceph-mon[23807]: osdmap e67: 8 total, 8 up, 8 in 2024-05-09T03:49:49.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:48 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-09T03:49:49.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:48 smithi078 ceph-mon[22905]: osdmap e67: 8 total, 8 up, 8 in 2024-05-09T03:49:49.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:48 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-09T03:49:49.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:48 smithi078 ceph-mon[27629]: osdmap e67: 8 total, 8 up, 8 in 2024-05-09T03:49:49.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:48 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-09T03:49:49.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 podman[47337]: 2024-05-09 03:49:49.015352542 +0000 UTC m=+0.637070787 container remove d1a5546ccafc09c24e60a31332a45ba24fc278718594628269df022c45cb07f0 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:49.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 podman[47337]: 2024-05-09 03:49:49.116172401 +0000 UTC m=+0.737890606 volume remove 49922e1faeac4d9a06e45c6fae0796f8acc64247ae7a6deb83d51f7cd02e3853 2024-05-09T03:49:49.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-09T03:49:49.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 systemd[1]: Stopped Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:49:49.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 systemd[1]: Starting Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:49:49.703 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 podman[47422]: 2024-05-09 03:49:49.24689848 +0000 UTC m=+0.017844886 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-05-09T03:49:49.703 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 podman[47422]: 2024-05-09 03:49:49.433788907 +0000 UTC m=+0.204735293 volume create 61d1f91c2fc39cda2db7a61508c6c2728a6e5b514eca5977fadfeccc7079faeb 2024-05-09T03:49:49.703 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 podman[47422]: 2024-05-09 03:49:49.54253619 +0000 UTC m=+0.313482578 container create 1bc0d1c4e0c8c5b06bdc493a68a7a354fc3b669398622db4a4cce63f62b2a47a (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:50.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:49 smithi132 ceph-mon[23807]: pgmap v58: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-05-09T03:49:50.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:49 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-05-09T03:49:50.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:49 smithi132 ceph-mon[23807]: osdmap e68: 8 total, 8 up, 8 in 2024-05-09T03:49:50.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:49 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-09T03:49:50.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:49 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:50.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-mon[22905]: pgmap v58: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-05-09T03:49:50.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-05-09T03:49:50.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-mon[22905]: osdmap e68: 8 total, 8 up, 8 in 2024-05-09T03:49:50.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-09T03:49:50.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:50.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:49 smithi078 ceph-mon[27629]: pgmap v58: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-05-09T03:49:50.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:49 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-05-09T03:49:50.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:49 smithi078 ceph-mon[27629]: osdmap e68: 8 total, 8 up, 8 in 2024-05-09T03:49:50.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:49 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-09T03:49:50.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:49 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:50.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 podman[47422]: 2024-05-09 03:49:49.803446071 +0000 UTC m=+0.574392467 container init 1bc0d1c4e0c8c5b06bdc493a68a7a354fc3b669398622db4a4cce63f62b2a47a (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:50.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 podman[47422]: 2024-05-09 03:49:49.806389558 +0000 UTC m=+0.577335948 container start 1bc0d1c4e0c8c5b06bdc493a68a7a354fc3b669398622db4a4cce63f62b2a47a (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:50.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=info ts=2024-05-09T03:49:49.816Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-05-09T03:49:50.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=info ts=2024-05-09T03:49:49.816Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-05-09T03:49:50.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=info ts=2024-05-09T03:49:49.818Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.78 port=9094 2024-05-09T03:49:50.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=info ts=2024-05-09T03:49:49.819Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-09T03:49:50.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=info ts=2024-05-09T03:49:49.843Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-09T03:49:50.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=info ts=2024-05-09T03:49:49.843Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-09T03:49:50.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=info ts=2024-05-09T03:49:49.846Z caller=main.go:518 msg=Listening address=:9093 2024-05-09T03:49:50.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=info ts=2024-05-09T03:49:49.846Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-05-09T03:49:50.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 bash[47422]: 1bc0d1c4e0c8c5b06bdc493a68a7a354fc3b669398622db4a4cce63f62b2a47a 2024-05-09T03:49:50.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:49 smithi078 systemd[1]: Started Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:49:51.032 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "id": "d657a640-c14a-45ae-b827-20bfc7bd7c00", 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "epoch": 1, 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "predecessor_uuid": "6dadd3f6-4934-4c46-821d-a410031d04a4", 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "sync_status": [], 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "period_map": { 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "id": "d657a640-c14a-45ae-b827-20bfc7bd7c00", 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "zonegroups": [ 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "id": "501e9ebe-1990-4eff-b292-80554f4357fe", 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "name": "default", 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "api_name": "default", 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "is_master": "true", 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "endpoints": [], 2024-05-09T03:49:51.033 INFO:teuthology.orchestra.run.smithi078.stdout: "hostnames": [], 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: "hostnames_s3website": [], 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: "master_zone": "374ba5e6-d9c9-4614-83a5-07436a4990b2", 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: "zones": [ 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: "id": "374ba5e6-d9c9-4614-83a5-07436a4990b2", 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: "name": "z", 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: "endpoints": [], 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: "log_meta": "false", 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: "log_data": "false", 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: "bucket_index_max_shards": 11, 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: "read_only": "false", 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: "tier_type": "", 2024-05-09T03:49:51.034 INFO:teuthology.orchestra.run.smithi078.stdout: "sync_from_all": "true", 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: "sync_from": [], 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: "redirect_zone": "" 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: "placement_targets": [ 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: "name": "default-placement", 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: "tags": [], 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: "storage_classes": [ 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: "STANDARD" 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-05-09T03:49:51.035 INFO:teuthology.orchestra.run.smithi078.stdout: "default_placement": "default-placement", 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: "realm_id": "0ae48363-ad56-463d-95e8-e141d876bdce", 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: "sync_policy": { 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: "groups": [] 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: "short_zone_ids": [ 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: "key": "374ba5e6-d9c9-4614-83a5-07436a4990b2", 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: "val": 396538892 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: "master_zonegroup": "501e9ebe-1990-4eff-b292-80554f4357fe", 2024-05-09T03:49:51.036 INFO:teuthology.orchestra.run.smithi078.stdout: "master_zone": "374ba5e6-d9c9-4614-83a5-07436a4990b2", 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "period_config": { 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "bucket_quota": { 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "enabled": false, 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "check_on_raw": false, 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "max_size": -1, 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "max_size_kb": 0, 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "max_objects": -1 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "user_quota": { 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "enabled": false, 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "check_on_raw": false, 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "max_size": -1, 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "max_size_kb": 0, 2024-05-09T03:49:51.037 INFO:teuthology.orchestra.run.smithi078.stdout: "max_objects": -1 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: "user_ratelimit": { 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: "max_read_ops": 0, 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: "max_write_ops": 0, 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: "max_read_bytes": 0, 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: "max_write_bytes": 0, 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: "enabled": false 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: "bucket_ratelimit": { 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: "max_read_ops": 0, 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: "max_write_ops": 0, 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: "max_read_bytes": 0, 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: "max_write_bytes": 0, 2024-05-09T03:49:51.038 INFO:teuthology.orchestra.run.smithi078.stdout: "enabled": false 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout: "anonymous_ratelimit": { 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout: "max_read_ops": 0, 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout: "max_write_ops": 0, 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout: "max_read_bytes": 0, 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout: "max_write_bytes": 0, 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout: "enabled": false 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout: "realm_id": "0ae48363-ad56-463d-95e8-e141d876bdce", 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout: "realm_name": "r", 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout: "realm_epoch": 2 2024-05-09T03:49:51.039 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T03:49:51.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:50 smithi132 ceph-mon[23807]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T03:49:51.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:50 smithi132 ceph-mon[23807]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T03:49:51.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:50 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-05-09T03:49:51.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:50 smithi132 ceph-mon[23807]: osdmap e69: 8 total, 8 up, 8 in 2024-05-09T03:49:51.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:50 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-09T03:49:51.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:50 smithi078 ceph-mon[22905]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T03:49:51.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:50 smithi078 ceph-mon[22905]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T03:49:51.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:50 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-05-09T03:49:51.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:50 smithi078 ceph-mon[22905]: osdmap e69: 8 total, 8 up, 8 in 2024-05-09T03:49:51.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:50 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-09T03:49:51.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:50 smithi078 ceph-mon[27629]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T03:49:51.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:50 smithi078 ceph-mon[27629]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T03:49:51.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:50 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-05-09T03:49:51.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:50 smithi078 ceph-mon[27629]: osdmap e69: 8 total, 8 up, 8 in 2024-05-09T03:49:51.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:50 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-09T03:49:51.932 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-05-09T03:49:52.241 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:51 smithi078 ceph-mon[22905]: pgmap v61: 129 pgs: 58 unknown, 71 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:52.241 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:51 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-05-09T03:49:52.241 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:51 smithi078 ceph-mon[22905]: osdmap e70: 8 total, 8 up, 8 in 2024-05-09T03:49:52.242 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:51 smithi078 ceph-mon[27629]: pgmap v61: 129 pgs: 58 unknown, 71 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:52.242 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:51 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-05-09T03:49:52.242 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:51 smithi078 ceph-mon[27629]: osdmap e70: 8 total, 8 up, 8 in 2024-05-09T03:49:52.242 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:51 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=info ts=2024-05-09T03:49:51.819Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000202s 2024-05-09T03:49:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:51 smithi132 ceph-mon[23807]: pgmap v61: 129 pgs: 58 unknown, 71 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:49:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:51 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/882658554' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-05-09T03:49:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:51 smithi132 ceph-mon[23807]: osdmap e70: 8 total, 8 up, 8 in 2024-05-09T03:49:52.931 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:49:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:49:52] "GET /metrics HTTP/1.1" 200 194209 "" "Prometheus/2.33.4" 2024-05-09T03:49:53.753 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:49:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:49:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:49:54.038 INFO:teuthology.orchestra.run.smithi078.stdout:Scheduled rgw.foo update... 2024-05-09T03:49:54.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:53 smithi132 ceph-mon[23807]: pgmap v63: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 224 B/s rd, 449 B/s wr, 1 op/s 2024-05-09T03:49:54.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:53 smithi078 ceph-mon[22905]: pgmap v63: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 224 B/s rd, 449 B/s wr, 1 op/s 2024-05-09T03:49:54.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:53 smithi078 ceph-mon[27629]: pgmap v63: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 224 B/s rd, 449 B/s wr, 1 op/s 2024-05-09T03:49:54.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph osd pool create foo' 2024-05-09T03:49:55.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:55 smithi078 ceph-mon[22905]: from='client.24586 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:49:55.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:55 smithi078 ceph-mon[22905]: Saving service rgw.foo spec with placement count:2 2024-05-09T03:49:55.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:55 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:55.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:55 smithi078 ceph-mon[27629]: from='client.24586 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:49:55.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:55 smithi078 ceph-mon[27629]: Saving service rgw.foo spec with placement count:2 2024-05-09T03:49:55.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:55 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:55 smithi132 ceph-mon[23807]: from='client.24586 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:49:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:55 smithi132 ceph-mon[23807]: Saving service rgw.foo spec with placement count:2 2024-05-09T03:49:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:55 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:49:56.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:56 smithi132 ceph-mon[23807]: pgmap v64: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-09T03:49:56.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:56 smithi078 ceph-mon[22905]: pgmap v64: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-09T03:49:56.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:56 smithi078 ceph-mon[27629]: pgmap v64: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-09T03:49:57.060 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'foo' created 2024-05-09T03:49:57.356 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:57 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2301364926' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-09T03:49:57.356 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:57 smithi132 ceph-mon[23807]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-09T03:49:57.386 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:57 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2301364926' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-09T03:49:57.386 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:57 smithi078 ceph-mon[22905]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-09T03:49:57.386 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:57 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2301364926' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-09T03:49:57.386 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:57 smithi078 ceph-mon[27629]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-09T03:49:57.795 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'rbd pool init foo' 2024-05-09T03:49:57.953 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 systemd[1]: Stopping Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:49:57.953 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 bash[40583]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus.a" found: no such container 2024-05-09T03:49:58.064 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:58 smithi078 ceph-mon[22905]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-05-09T03:49:58.064 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:58 smithi078 ceph-mon[22905]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-05-09T03:49:58.064 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:58 smithi078 ceph-mon[22905]: osdmap e71: 8 total, 8 up, 8 in 2024-05-09T03:49:58.252 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:58 smithi132 ceph-mon[23807]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-05-09T03:49:58.252 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:58 smithi132 ceph-mon[23807]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-05-09T03:49:58.252 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:58 smithi132 ceph-mon[23807]: osdmap e71: 8 total, 8 up, 8 in 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:49:57.952Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:49:57.952Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:49:57.952Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:49:57.952Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:49:57.952Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:49:57.952Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:49:57.952Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:49:57.952Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:49:57.952Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:49:57.953Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:49:57.953Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[38889]: ts=2024-05-09T03:49:57.953Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-09T03:49:58.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:57 smithi132 podman[40596]: 2024-05-09 03:49:57.962617821 +0000 UTC m=+0.145101413 container died c9092d8a07e890fc497ebf547a4d39d2dbe86c2121a10129df1374d169ca06db (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:58.398 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:58 smithi078 ceph-mon[27629]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-05-09T03:49:58.398 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:58 smithi078 ceph-mon[27629]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-05-09T03:49:58.398 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:58 smithi078 ceph-mon[27629]: osdmap e71: 8 total, 8 up, 8 in 2024-05-09T03:49:58.504 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:58 smithi132 podman[40596]: 2024-05-09 03:49:58.351326656 +0000 UTC m=+0.533810257 container cleanup c9092d8a07e890fc497ebf547a4d39d2dbe86c2121a10129df1374d169ca06db (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:58.504 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:58 smithi132 bash[40596]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a 2024-05-09T03:49:58.504 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:58 smithi132 bash[40623]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus.a" found: no such container 2024-05-09T03:49:58.947 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:58 smithi132 podman[40611]: 2024-05-09 03:49:58.610921149 +0000 UTC m=+0.645080594 container remove c9092d8a07e890fc497ebf547a4d39d2dbe86c2121a10129df1374d169ca06db (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:58.947 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:58 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-09T03:49:58.947 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:58 smithi132 systemd[1]: Stopped Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:49:58.947 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:58 smithi132 systemd[1]: Starting Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:49:58.947 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:58 smithi132 podman[40691]: 2024-05-09 03:49:58.75300928 +0000 UTC m=+0.019170800 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-09T03:49:58.947 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:58 smithi132 podman[40691]: 2024-05-09 03:49:58.853404413 +0000 UTC m=+0.119565916 container create 69b608c5f2ac802d5bc4fa19119d4e3ba140b2f223b30768ee1ca9c12d8cb605 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:59.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:49:59 smithi078 ceph-mon[22905]: osdmap e72: 8 total, 8 up, 8 in 2024-05-09T03:49:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:49:59 smithi078 ceph-mon[27629]: osdmap e72: 8 total, 8 up, 8 in 2024-05-09T03:49:59.306 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:49:59 smithi132 ceph-mon[23807]: osdmap e72: 8 total, 8 up, 8 in 2024-05-09T03:49:59.306 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 podman[40691]: 2024-05-09 03:49:59.055533854 +0000 UTC m=+0.321695365 container init 69b608c5f2ac802d5bc4fa19119d4e3ba140b2f223b30768ee1ca9c12d8cb605 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:59.306 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 podman[40691]: 2024-05-09 03:49:59.058236323 +0000 UTC m=+0.324397827 container start 69b608c5f2ac802d5bc4fa19119d4e3ba140b2f223b30768ee1ca9c12d8cb605 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:49:59.306 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.082Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-09T03:49:59.306 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.082Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-09T03:49:59.306 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.082Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-09T03:49:59.306 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.082Z caller=main.go:518 level=info host_details="(Linux 5.14.0-444.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024 x86_64 smithi132 (none))" 2024-05-09T03:49:59.306 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.082Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-09T03:49:59.306 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.082Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-09T03:49:59.307 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.084Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-09T03:49:59.307 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.084Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-09T03:49:59.307 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.085Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-09T03:49:59.307 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.086Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-09T03:49:59.307 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.087Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.468µs 2024-05-09T03:49:59.307 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.087Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-09T03:49:59.307 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 bash[40691]: 69b608c5f2ac802d5bc4fa19119d4e3ba140b2f223b30768ee1ca9c12d8cb605 2024-05-09T03:49:59.307 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 systemd[1]: Started Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:49:59.307 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.305Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-05-09T03:49:59.307 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.305Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-05-09T03:49:59.307 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.305Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.165µs wal_replay_duration=218.616195ms total_replay_duration=218.651778ms 2024-05-09T03:49:59.751 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.307Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-09T03:49:59.751 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.307Z caller=main.go:947 level=info msg="TSDB started" 2024-05-09T03:49:59.751 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.307Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-09T03:49:59.751 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.322Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.114744ms db_storage=1.315µs remote_storage=1.315µs web_handler=381ns query_engine=735ns scrape=432.486µs scrape_sd=35.076µs notify=29.106µs notify_sd=14.71µs rules=14.34709ms 2024-05-09T03:49:59.751 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:49:59 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:49:59.322Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-09T03:50:00.044 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:49:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=info ts=2024-05-09T03:49:59.821Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002221102s 2024-05-09T03:50:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: pgmap v68: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 949 B/s wr, 2 op/s 2024-05-09T03:50:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: osdmap e73: 8 total, 8 up, 8 in 2024-05-09T03:50:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.132:9095"}]: dispatch 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.132:9095"}]: dispatch 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.132:3000"}]: dispatch 2024-05-09T03:50:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.132:3000"}]: dispatch 2024-05-09T03:50:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:50:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:50:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/103797737' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-09T03:50:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:00 smithi132 ceph-mon[23807]: overall HEALTH_OK 2024-05-09T03:50:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: pgmap v68: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 949 B/s wr, 2 op/s 2024-05-09T03:50:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: osdmap e73: 8 total, 8 up, 8 in 2024-05-09T03:50:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T03:50:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T03:50:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-05-09T03:50:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-05-09T03:50:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T03:50:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.132:9095"}]: dispatch 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.132:9095"}]: dispatch 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.132:3000"}]: dispatch 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.132:3000"}]: dispatch 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/103797737' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-09T03:50:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[22905]: overall HEALTH_OK 2024-05-09T03:50:00.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: pgmap v68: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 949 B/s wr, 2 op/s 2024-05-09T03:50:00.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: osdmap e73: 8 total, 8 up, 8 in 2024-05-09T03:50:00.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:00.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T03:50:00.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T03:50:00.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-05-09T03:50:00.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-05-09T03:50:00.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:00.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T03:50:00.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T03:50:00.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.132:9095"}]: dispatch 2024-05-09T03:50:00.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.132:9095"}]: dispatch 2024-05-09T03:50:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T03:50:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T03:50:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.132:3000"}]: dispatch 2024-05-09T03:50:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.132:3000"}]: dispatch 2024-05-09T03:50:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:50:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:50:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/103797737' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-09T03:50:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:00 smithi078 ceph-mon[27629]: overall HEALTH_OK 2024-05-09T03:50:01.238 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:01 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/103797737' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-05-09T03:50:01.238 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:01 smithi078 ceph-mon[22905]: osdmap e74: 8 total, 8 up, 8 in 2024-05-09T03:50:01.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:01 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/103797737' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-05-09T03:50:01.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:01 smithi132 ceph-mon[23807]: osdmap e74: 8 total, 8 up, 8 in 2024-05-09T03:50:01.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:01 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/103797737' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-05-09T03:50:01.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:01 smithi078 ceph-mon[27629]: osdmap e74: 8 total, 8 up, 8 in 2024-05-09T03:50:02.328 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:02 smithi132 ceph-mon[23807]: pgmap v71: 161 pgs: 14 unknown, 147 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:50:02.328 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:02 smithi132 ceph-mon[23807]: osdmap e75: 8 total, 8 up, 8 in 2024-05-09T03:50:02.328 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:02 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:02.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:02 smithi078 ceph-mon[22905]: pgmap v71: 161 pgs: 14 unknown, 147 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:50:02.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:02 smithi078 ceph-mon[22905]: osdmap e75: 8 total, 8 up, 8 in 2024-05-09T03:50:02.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:02 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:02 smithi078 ceph-mon[27629]: pgmap v71: 161 pgs: 14 unknown, 147 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:50:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:02 smithi078 ceph-mon[27629]: osdmap e75: 8 total, 8 up, 8 in 2024-05-09T03:50:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:02 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:02.942 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch apply iscsi foo u p' 2024-05-09T03:50:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:03 smithi132 ceph-mon[23807]: osdmap e76: 8 total, 8 up, 8 in 2024-05-09T03:50:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:03 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:03 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:03 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:03 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:03 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:03 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T03:50:03.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:03 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T03:50:03.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:03 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-09T03:50:03.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:03 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:03 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:03.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[22905]: osdmap e76: 8 total, 8 up, 8 in 2024-05-09T03:50:03.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T03:50:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T03:50:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-09T03:50:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:03.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[27629]: osdmap e76: 8 total, 8 up, 8 in 2024-05-09T03:50:03.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T03:50:03.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T03:50:03.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-09T03:50:03.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:03.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:03 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:04.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:04 smithi132 ceph-mon[23807]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:50:04.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:04 smithi132 ceph-mon[23807]: Saving service rgw.foo spec with placement count:2 2024-05-09T03:50:04.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:04 smithi132 ceph-mon[23807]: Deploying daemon rgw.foo.smithi078.bfidtv on smithi078 2024-05-09T03:50:04.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:04 smithi078 ceph-mon[22905]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:50:04.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:04 smithi078 ceph-mon[22905]: Saving service rgw.foo spec with placement count:2 2024-05-09T03:50:04.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:04 smithi078 ceph-mon[22905]: Deploying daemon rgw.foo.smithi078.bfidtv on smithi078 2024-05-09T03:50:04.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:04 smithi078 ceph-mon[27629]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:50:04.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:04 smithi078 ceph-mon[27629]: Saving service rgw.foo spec with placement count:2 2024-05-09T03:50:04.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:04 smithi078 ceph-mon[27629]: Deploying daemon rgw.foo.smithi078.bfidtv on smithi078 2024-05-09T03:50:05.514 INFO:teuthology.orchestra.run.smithi078.stdout:Scheduled iscsi.foo update... 2024-05-09T03:50:06.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:06 smithi132 ceph-mon[23807]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:50:06.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:06 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:06.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:06 smithi078 ceph-mon[22905]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:50:06.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:06 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:06 smithi078 ceph-mon[27629]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:50:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:06 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:06.622 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'sleep 180' 2024-05-09T03:50:07.436 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:07 smithi078 ceph-mon[22905]: from='client.24604 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:50:07.436 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:07 smithi078 ceph-mon[22905]: Saving service iscsi.foo spec with placement count:1 2024-05-09T03:50:07.436 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:07 smithi078 ceph-mon[27629]: from='client.24604 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:50:07.436 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:07 smithi078 ceph-mon[27629]: Saving service iscsi.foo spec with placement count:1 2024-05-09T03:50:07.747 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:07 smithi132 ceph-mon[23807]: from='client.24604 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:50:07.747 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:07 smithi132 ceph-mon[23807]: Saving service iscsi.foo spec with placement count:1 2024-05-09T03:50:08.392 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[22905]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 325 B/s wr, 0 op/s 2024-05-09T03:50:08.392 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:08.392 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T03:50:08.392 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T03:50:08.393 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-09T03:50:08.393 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:08.393 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:08.393 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[22905]: Deploying daemon rgw.foo.smithi132.pcutwm on smithi132 2024-05-09T03:50:08.393 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[22905]: osdmap e77: 8 total, 8 up, 8 in 2024-05-09T03:50:08.393 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[27629]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 325 B/s wr, 0 op/s 2024-05-09T03:50:08.393 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:08.393 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T03:50:08.393 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T03:50:08.393 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-09T03:50:08.393 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:08.393 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:08.394 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[27629]: Deploying daemon rgw.foo.smithi132.pcutwm on smithi132 2024-05-09T03:50:08.394 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:08 smithi078 ceph-mon[27629]: osdmap e77: 8 total, 8 up, 8 in 2024-05-09T03:50:08.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:08 smithi132 ceph-mon[23807]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 325 B/s wr, 0 op/s 2024-05-09T03:50:08.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:08 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:08.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:08 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T03:50:08.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:08 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T03:50:08.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:08 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-09T03:50:08.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:08 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:08.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:08 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:08.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:08 smithi132 ceph-mon[23807]: Deploying daemon rgw.foo.smithi132.pcutwm on smithi132 2024-05-09T03:50:08.666 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:08 smithi132 ceph-mon[23807]: osdmap e77: 8 total, 8 up, 8 in 2024-05-09T03:50:10.572 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:10 smithi132 ceph-mon[23807]: pgmap v78: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 281 B/s wr, 0 op/s 2024-05-09T03:50:10.572 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:10 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:10.573 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:10 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:50:10.573 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:10 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:10.573 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:10 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:50:10.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:10 smithi078 ceph-mon[22905]: pgmap v78: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 281 B/s wr, 0 op/s 2024-05-09T03:50:10.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:10 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:10.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:10 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:50:10.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:10 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:10.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:10 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:50:10.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:10 smithi078 ceph-mon[27629]: pgmap v78: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 281 B/s wr, 0 op/s 2024-05-09T03:50:10.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:10 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:10.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:10 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:50:10.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:10 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:10.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:10 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:50:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:12 smithi132 ceph-mon[23807]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 192 KiB/s rd, 4.3 KiB/s wr, 349 op/s 2024-05-09T03:50:12.677 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:12 smithi078 ceph-mon[22905]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 192 KiB/s rd, 4.3 KiB/s wr, 349 op/s 2024-05-09T03:50:12.677 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:12 smithi078 ceph-mon[27629]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 192 KiB/s rd, 4.3 KiB/s wr, 349 op/s 2024-05-09T03:50:13.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:50:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:50:12] "GET /metrics HTTP/1.1" 200 197895 "" "Prometheus/2.33.4" 2024-05-09T03:50:13.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:50:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:50:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:50:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:13 smithi132 ceph-mon[23807]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 159 KiB/s rd, 3.6 KiB/s wr, 289 op/s 2024-05-09T03:50:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:13 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:13 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:13.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:13 smithi132 ceph-mon[23807]: Checking dashboard <-> RGW credentials 2024-05-09T03:50:14.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:13 smithi078 ceph-mon[22905]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 159 KiB/s rd, 3.6 KiB/s wr, 289 op/s 2024-05-09T03:50:14.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:13 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:14.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:13 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:14.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:13 smithi078 ceph-mon[22905]: Checking dashboard <-> RGW credentials 2024-05-09T03:50:14.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:13 smithi078 ceph-mon[27629]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 159 KiB/s rd, 3.6 KiB/s wr, 289 op/s 2024-05-09T03:50:14.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:13 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:14.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:13 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:14.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:13 smithi078 ceph-mon[27629]: Checking dashboard <-> RGW credentials 2024-05-09T03:50:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: Checking pool "foo" exists for service iscsi.foo 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: Deploying daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-05-09T03:50:15.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-09T03:50:15.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-09T03:50:15.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:50:15.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:50:15.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:50:15.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:50:15.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:14 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:15.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:15.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:15.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:15.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: Checking pool "foo" exists for service iscsi.foo 2024-05-09T03:50:15.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:50:15.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:50:15.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-05-09T03:50:15.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:15.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: Deploying daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:50:15.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-05-09T03:50:15.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-05-09T03:50:15.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-05-09T03:50:15.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-09T03:50:15.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-09T03:50:15.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-05-09T03:50:15.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-09T03:50:15.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-09T03:50:15.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:50:15.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:50:15.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:50:15.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:50:15.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:15.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:15.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:15.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:15.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: Checking pool "foo" exists for service iscsi.foo 2024-05-09T03:50:15.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: Deploying daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:50:15.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:50:15.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:50:15.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:50:15.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:14 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:16.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:15 smithi132 ceph-mon[23807]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 159 KiB/s rd, 3.6 KiB/s wr, 289 op/s 2024-05-09T03:50:16.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]': finished 2024-05-09T03:50:16.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-05-09T03:50:16.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-05-09T03:50:16.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-05-09T03:50:16.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:15 smithi132 ceph-mon[23807]: osdmap e78: 8 total, 8 up, 8 in 2024-05-09T03:50:16.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:15 smithi132 ceph-mon[23807]: mgrmap e20: y(active, since 2m), standbys: x 2024-05-09T03:50:16.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[27629]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 159 KiB/s rd, 3.6 KiB/s wr, 289 op/s 2024-05-09T03:50:16.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]': finished 2024-05-09T03:50:16.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-05-09T03:50:16.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-05-09T03:50:16.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-05-09T03:50:16.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[27629]: osdmap e78: 8 total, 8 up, 8 in 2024-05-09T03:50:16.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[27629]: mgrmap e20: y(active, since 2m), standbys: x 2024-05-09T03:50:16.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[22905]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 159 KiB/s rd, 3.6 KiB/s wr, 289 op/s 2024-05-09T03:50:16.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]': finished 2024-05-09T03:50:16.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-05-09T03:50:16.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-05-09T03:50:16.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-05-09T03:50:16.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[22905]: osdmap e78: 8 total, 8 up, 8 in 2024-05-09T03:50:16.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:15 smithi078 ceph-mon[22905]: mgrmap e20: y(active, since 2m), standbys: x 2024-05-09T03:50:17.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:16 smithi132 ceph-mon[23807]: osdmap e79: 8 total, 8 up, 8 in 2024-05-09T03:50:17.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:16 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:17.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:16 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:50:17.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:16 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:17.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:16 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:50:17.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:16 smithi078 ceph-mon[22905]: osdmap e79: 8 total, 8 up, 8 in 2024-05-09T03:50:17.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:16 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:17.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:16 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:50:17.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:16 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:17.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:16 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:50:17.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:16 smithi078 ceph-mon[27629]: osdmap e79: 8 total, 8 up, 8 in 2024-05-09T03:50:17.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:16 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:17.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:16 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:50:17.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:16 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:17.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:16 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:50:18.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:17 smithi132 ceph-mon[23807]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 235 KiB/s rd, 4.5 KiB/s wr, 413 op/s 2024-05-09T03:50:18.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:17 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3190529355' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T03:50:18.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:17 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1090772070' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1941182422"}]: dispatch 2024-05-09T03:50:18.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:17 smithi078 ceph-mon[27629]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 235 KiB/s rd, 4.5 KiB/s wr, 413 op/s 2024-05-09T03:50:18.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:17 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3190529355' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T03:50:18.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:17 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1090772070' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1941182422"}]: dispatch 2024-05-09T03:50:18.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:17 smithi078 ceph-mon[22905]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 235 KiB/s rd, 4.5 KiB/s wr, 413 op/s 2024-05-09T03:50:18.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:17 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3190529355' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T03:50:18.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:17 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1090772070' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1941182422"}]: dispatch 2024-05-09T03:50:18.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:18 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1090772070' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1941182422"}]': finished 2024-05-09T03:50:18.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:18 smithi078 ceph-mon[27629]: osdmap e80: 8 total, 8 up, 8 in 2024-05-09T03:50:18.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:18 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3466315862' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3325882269"}]: dispatch 2024-05-09T03:50:18.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:18 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3325882269"}]: dispatch 2024-05-09T03:50:18.969 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:18 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1090772070' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1941182422"}]': finished 2024-05-09T03:50:18.969 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:18 smithi078 ceph-mon[22905]: osdmap e80: 8 total, 8 up, 8 in 2024-05-09T03:50:18.969 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:18 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3466315862' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3325882269"}]: dispatch 2024-05-09T03:50:18.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:18 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3325882269"}]: dispatch 2024-05-09T03:50:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:18 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1090772070' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1941182422"}]': finished 2024-05-09T03:50:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:18 smithi132 ceph-mon[23807]: osdmap e80: 8 total, 8 up, 8 in 2024-05-09T03:50:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:18 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3466315862' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3325882269"}]: dispatch 2024-05-09T03:50:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:18 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3325882269"}]: dispatch 2024-05-09T03:50:20.222 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:19 smithi078 ceph-mon[22905]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 341 B/s wr, 69 op/s 2024-05-09T03:50:20.222 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:19 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3325882269"}]': finished 2024-05-09T03:50:20.223 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:19 smithi078 ceph-mon[22905]: osdmap e81: 8 total, 8 up, 8 in 2024-05-09T03:50:20.223 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:19 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3719227666' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1457479635"}]: dispatch 2024-05-09T03:50:20.223 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:19 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1457479635"}]: dispatch 2024-05-09T03:50:20.223 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:19 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:20.223 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:19 smithi078 ceph-mon[27629]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 341 B/s wr, 69 op/s 2024-05-09T03:50:20.223 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:19 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3325882269"}]': finished 2024-05-09T03:50:20.223 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:19 smithi078 ceph-mon[27629]: osdmap e81: 8 total, 8 up, 8 in 2024-05-09T03:50:20.223 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:19 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3719227666' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1457479635"}]: dispatch 2024-05-09T03:50:20.223 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:19 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1457479635"}]: dispatch 2024-05-09T03:50:20.223 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:19 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:20.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:19 smithi132 ceph-mon[23807]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 341 B/s wr, 69 op/s 2024-05-09T03:50:20.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:19 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3325882269"}]': finished 2024-05-09T03:50:20.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:19 smithi132 ceph-mon[23807]: osdmap e81: 8 total, 8 up, 8 in 2024-05-09T03:50:20.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:19 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3719227666' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1457479635"}]: dispatch 2024-05-09T03:50:20.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:19 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1457479635"}]: dispatch 2024-05-09T03:50:20.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:19 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1457479635"}]': finished 2024-05-09T03:50:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: osdmap e82: 8 total, 8 up, 8 in 2024-05-09T03:50:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T03:50:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T03:50:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: Adding iSCSI gateway http://:@172.21.15.78:5000 to Dashboard 2024-05-09T03:50:21.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T03:50:21.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T03:50:21.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T03:50:21.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T03:50:21.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:50:21.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:21.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:50:21.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: Checking dashboard <-> RGW credentials 2024-05-09T03:50:21.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3355210697' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3439374234"}]: dispatch 2024-05-09T03:50:21.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:20 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1457479635"}]': finished 2024-05-09T03:50:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: osdmap e82: 8 total, 8 up, 8 in 2024-05-09T03:50:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T03:50:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: Adding iSCSI gateway http://:@172.21.15.78:5000 to Dashboard 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: Checking dashboard <-> RGW credentials 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3355210697' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3439374234"}]: dispatch 2024-05-09T03:50:21.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1457479635"}]': finished 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: osdmap e82: 8 total, 8 up, 8 in 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: Adding iSCSI gateway http://:@172.21.15.78:5000 to Dashboard 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T03:50:21.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:21.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:50:21.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:50:21.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:50:21.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: Checking dashboard <-> RGW credentials 2024-05-09T03:50:21.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3355210697' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3439374234"}]: dispatch 2024-05-09T03:50:21.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:20 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:50:22.677 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:22 smithi078 ceph-mon[27629]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 212 B/s wr, 9 op/s; 9 B/s, 0 objects/s recovering 2024-05-09T03:50:22.677 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:22 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3355210697' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3439374234"}]': finished 2024-05-09T03:50:22.677 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:22 smithi078 ceph-mon[27629]: osdmap e83: 8 total, 8 up, 8 in 2024-05-09T03:50:22.677 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:22 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1062573006' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/987239519"}]: dispatch 2024-05-09T03:50:22.678 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:22 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/987239519"}]: dispatch 2024-05-09T03:50:22.678 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:22 smithi078 ceph-mon[22905]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 212 B/s wr, 9 op/s; 9 B/s, 0 objects/s recovering 2024-05-09T03:50:22.678 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:22 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3355210697' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3439374234"}]': finished 2024-05-09T03:50:22.678 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:22 smithi078 ceph-mon[22905]: osdmap e83: 8 total, 8 up, 8 in 2024-05-09T03:50:22.678 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:22 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1062573006' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/987239519"}]: dispatch 2024-05-09T03:50:22.678 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:22 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/987239519"}]: dispatch 2024-05-09T03:50:22.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:22 smithi132 ceph-mon[23807]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 212 B/s wr, 9 op/s; 9 B/s, 0 objects/s recovering 2024-05-09T03:50:22.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:22 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3355210697' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3439374234"}]': finished 2024-05-09T03:50:22.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:22 smithi132 ceph-mon[23807]: osdmap e83: 8 total, 8 up, 8 in 2024-05-09T03:50:22.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:22 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1062573006' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/987239519"}]: dispatch 2024-05-09T03:50:22.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:22 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/987239519"}]: dispatch 2024-05-09T03:50:23.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:50:22 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:50:22] "GET /metrics HTTP/1.1" 200 207323 "" "Prometheus/2.33.4" 2024-05-09T03:50:23.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:50:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:50:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:50:23.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:23 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/987239519"}]': finished 2024-05-09T03:50:23.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:23 smithi132 ceph-mon[23807]: osdmap e84: 8 total, 8 up, 8 in 2024-05-09T03:50:23.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:23 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3338653696' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/987239519"}]: dispatch 2024-05-09T03:50:23.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:23 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/987239519"}]: dispatch 2024-05-09T03:50:23.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:23 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/987239519"}]': finished 2024-05-09T03:50:23.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:23 smithi078 ceph-mon[27629]: osdmap e84: 8 total, 8 up, 8 in 2024-05-09T03:50:23.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:23 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3338653696' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/987239519"}]: dispatch 2024-05-09T03:50:23.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:23 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/987239519"}]: dispatch 2024-05-09T03:50:23.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:23 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/987239519"}]': finished 2024-05-09T03:50:23.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:23 smithi078 ceph-mon[22905]: osdmap e84: 8 total, 8 up, 8 in 2024-05-09T03:50:23.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:23 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3338653696' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/987239519"}]: dispatch 2024-05-09T03:50:23.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:23 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/987239519"}]: dispatch 2024-05-09T03:50:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:24 smithi132 ceph-mon[23807]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 9.0 KiB/s rd, 255 B/s wr, 11 op/s; 11 B/s, 0 objects/s recovering 2024-05-09T03:50:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:24 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/987239519"}]': finished 2024-05-09T03:50:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:24 smithi132 ceph-mon[23807]: osdmap e85: 8 total, 8 up, 8 in 2024-05-09T03:50:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:24 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2952794977' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/169193258"}]: dispatch 2024-05-09T03:50:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:24 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/169193258"}]: dispatch 2024-05-09T03:50:24.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:24 smithi078 ceph-mon[27629]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 9.0 KiB/s rd, 255 B/s wr, 11 op/s; 11 B/s, 0 objects/s recovering 2024-05-09T03:50:24.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:24 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/987239519"}]': finished 2024-05-09T03:50:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:24 smithi078 ceph-mon[27629]: osdmap e85: 8 total, 8 up, 8 in 2024-05-09T03:50:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:24 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2952794977' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/169193258"}]: dispatch 2024-05-09T03:50:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:24 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/169193258"}]: dispatch 2024-05-09T03:50:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:24 smithi078 ceph-mon[22905]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 9.0 KiB/s rd, 255 B/s wr, 11 op/s; 11 B/s, 0 objects/s recovering 2024-05-09T03:50:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:24 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/987239519"}]': finished 2024-05-09T03:50:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:24 smithi078 ceph-mon[22905]: osdmap e85: 8 total, 8 up, 8 in 2024-05-09T03:50:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:24 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2952794977' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/169193258"}]: dispatch 2024-05-09T03:50:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:24 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/169193258"}]: dispatch 2024-05-09T03:50:25.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:25 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/169193258"}]': finished 2024-05-09T03:50:25.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:25 smithi132 ceph-mon[23807]: osdmap e86: 8 total, 8 up, 8 in 2024-05-09T03:50:25.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:25 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2921597694' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1457479635"}]: dispatch 2024-05-09T03:50:25.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:25 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1457479635"}]: dispatch 2024-05-09T03:50:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:25 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/169193258"}]': finished 2024-05-09T03:50:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:25 smithi078 ceph-mon[22905]: osdmap e86: 8 total, 8 up, 8 in 2024-05-09T03:50:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:25 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2921597694' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1457479635"}]: dispatch 2024-05-09T03:50:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:25 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1457479635"}]: dispatch 2024-05-09T03:50:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:25 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/169193258"}]': finished 2024-05-09T03:50:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:25 smithi078 ceph-mon[27629]: osdmap e86: 8 total, 8 up, 8 in 2024-05-09T03:50:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:25 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2921597694' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1457479635"}]: dispatch 2024-05-09T03:50:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:25 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1457479635"}]: dispatch 2024-05-09T03:50:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:26 smithi132 ceph-mon[23807]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:50:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:26 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1457479635"}]': finished 2024-05-09T03:50:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:26 smithi132 ceph-mon[23807]: osdmap e87: 8 total, 8 up, 8 in 2024-05-09T03:50:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:26 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2600580495' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/169193258"}]: dispatch 2024-05-09T03:50:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:26 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/169193258"}]: dispatch 2024-05-09T03:50:26.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:26 smithi078 ceph-mon[22905]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:50:26.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:26 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1457479635"}]': finished 2024-05-09T03:50:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:26 smithi078 ceph-mon[22905]: osdmap e87: 8 total, 8 up, 8 in 2024-05-09T03:50:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:26 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2600580495' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/169193258"}]: dispatch 2024-05-09T03:50:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:26 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/169193258"}]: dispatch 2024-05-09T03:50:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:26 smithi078 ceph-mon[27629]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:50:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:26 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1457479635"}]': finished 2024-05-09T03:50:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:26 smithi078 ceph-mon[27629]: osdmap e87: 8 total, 8 up, 8 in 2024-05-09T03:50:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:26 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2600580495' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/169193258"}]: dispatch 2024-05-09T03:50:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:26 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/169193258"}]: dispatch 2024-05-09T03:50:27.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:27 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/169193258"}]': finished 2024-05-09T03:50:27.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:27 smithi132 ceph-mon[23807]: osdmap e88: 8 total, 8 up, 8 in 2024-05-09T03:50:27.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:27 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2922524592' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/816857241"}]: dispatch 2024-05-09T03:50:27.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:27 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/169193258"}]': finished 2024-05-09T03:50:27.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:27 smithi078 ceph-mon[22905]: osdmap e88: 8 total, 8 up, 8 in 2024-05-09T03:50:27.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:27 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2922524592' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/816857241"}]: dispatch 2024-05-09T03:50:27.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:27 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/169193258"}]': finished 2024-05-09T03:50:27.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:27 smithi078 ceph-mon[27629]: osdmap e88: 8 total, 8 up, 8 in 2024-05-09T03:50:27.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:27 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2922524592' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/816857241"}]: dispatch 2024-05-09T03:50:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:28 smithi132 ceph-mon[23807]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:28 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:50:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:28 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2922524592' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/816857241"}]': finished 2024-05-09T03:50:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:28 smithi132 ceph-mon[23807]: osdmap e89: 8 total, 8 up, 8 in 2024-05-09T03:50:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:28 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2019537315' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1125963846"}]: dispatch 2024-05-09T03:50:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:28 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1125963846"}]: dispatch 2024-05-09T03:50:28.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:28 smithi078 ceph-mon[27629]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:28 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:50:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:28 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2922524592' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/816857241"}]': finished 2024-05-09T03:50:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:28 smithi078 ceph-mon[27629]: osdmap e89: 8 total, 8 up, 8 in 2024-05-09T03:50:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:28 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2019537315' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1125963846"}]: dispatch 2024-05-09T03:50:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:28 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1125963846"}]: dispatch 2024-05-09T03:50:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:28 smithi078 ceph-mon[22905]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:28 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:50:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:28 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2922524592' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/816857241"}]': finished 2024-05-09T03:50:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:28 smithi078 ceph-mon[22905]: osdmap e89: 8 total, 8 up, 8 in 2024-05-09T03:50:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:28 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2019537315' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1125963846"}]: dispatch 2024-05-09T03:50:28.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:28 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1125963846"}]: dispatch 2024-05-09T03:50:29.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:29 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1125963846"}]': finished 2024-05-09T03:50:29.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:29 smithi078 ceph-mon[27629]: osdmap e90: 8 total, 8 up, 8 in 2024-05-09T03:50:29.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:29 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/83339944' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3754415003"}]: dispatch 2024-05-09T03:50:29.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:29 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3754415003"}]: dispatch 2024-05-09T03:50:29.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:29 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1125963846"}]': finished 2024-05-09T03:50:29.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:29 smithi078 ceph-mon[22905]: osdmap e90: 8 total, 8 up, 8 in 2024-05-09T03:50:29.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:29 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/83339944' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3754415003"}]: dispatch 2024-05-09T03:50:29.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:29 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3754415003"}]: dispatch 2024-05-09T03:50:30.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:29 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1125963846"}]': finished 2024-05-09T03:50:30.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:29 smithi132 ceph-mon[23807]: osdmap e90: 8 total, 8 up, 8 in 2024-05-09T03:50:30.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:29 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/83339944' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3754415003"}]: dispatch 2024-05-09T03:50:30.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:29 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3754415003"}]: dispatch 2024-05-09T03:50:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:30 smithi132 ceph-mon[23807]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:30 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3754415003"}]': finished 2024-05-09T03:50:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:30 smithi132 ceph-mon[23807]: osdmap e91: 8 total, 8 up, 8 in 2024-05-09T03:50:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:30 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2066569366' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/906720789"}]: dispatch 2024-05-09T03:50:31.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:30 smithi078 ceph-mon[22905]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:31.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:30 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3754415003"}]': finished 2024-05-09T03:50:31.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:30 smithi078 ceph-mon[22905]: osdmap e91: 8 total, 8 up, 8 in 2024-05-09T03:50:31.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:30 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2066569366' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/906720789"}]: dispatch 2024-05-09T03:50:31.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:30 smithi078 ceph-mon[27629]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:31.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:30 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3754415003"}]': finished 2024-05-09T03:50:31.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:30 smithi078 ceph-mon[27629]: osdmap e91: 8 total, 8 up, 8 in 2024-05-09T03:50:31.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:30 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2066569366' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/906720789"}]: dispatch 2024-05-09T03:50:32.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:31 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2066569366' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/906720789"}]': finished 2024-05-09T03:50:32.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:31 smithi132 ceph-mon[23807]: osdmap e92: 8 total, 8 up, 8 in 2024-05-09T03:50:32.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:31 smithi132 ceph-mon[23807]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:32.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:31 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2981562923' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2700265794"}]: dispatch 2024-05-09T03:50:32.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:31 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2066569366' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/906720789"}]': finished 2024-05-09T03:50:32.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:31 smithi078 ceph-mon[22905]: osdmap e92: 8 total, 8 up, 8 in 2024-05-09T03:50:32.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:31 smithi078 ceph-mon[22905]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:32.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:31 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2981562923' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2700265794"}]: dispatch 2024-05-09T03:50:32.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:31 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2066569366' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/906720789"}]': finished 2024-05-09T03:50:32.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:31 smithi078 ceph-mon[27629]: osdmap e92: 8 total, 8 up, 8 in 2024-05-09T03:50:32.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:31 smithi078 ceph-mon[27629]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:32.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:31 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2981562923' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2700265794"}]: dispatch 2024-05-09T03:50:32.905 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:32 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2981562923' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2700265794"}]': finished 2024-05-09T03:50:32.905 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:32 smithi132 ceph-mon[23807]: osdmap e93: 8 total, 8 up, 8 in 2024-05-09T03:50:32.905 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:32 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1815570086' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/147802961"}]: dispatch 2024-05-09T03:50:32.905 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:32 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/147802961"}]: dispatch 2024-05-09T03:50:33.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:32 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2981562923' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2700265794"}]': finished 2024-05-09T03:50:33.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:32 smithi078 ceph-mon[22905]: osdmap e93: 8 total, 8 up, 8 in 2024-05-09T03:50:33.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:32 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1815570086' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/147802961"}]: dispatch 2024-05-09T03:50:33.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:32 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/147802961"}]: dispatch 2024-05-09T03:50:33.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:50:32 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:50:32] "GET /metrics HTTP/1.1" 200 208159 "" "Prometheus/2.33.4" 2024-05-09T03:50:33.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:32 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2981562923' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2700265794"}]': finished 2024-05-09T03:50:33.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:32 smithi078 ceph-mon[27629]: osdmap e93: 8 total, 8 up, 8 in 2024-05-09T03:50:33.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:32 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1815570086' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/147802961"}]: dispatch 2024-05-09T03:50:33.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:32 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/147802961"}]: dispatch 2024-05-09T03:50:33.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:50:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:50:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:50:34.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:33 smithi132 ceph-mon[23807]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:33 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/147802961"}]': finished 2024-05-09T03:50:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:33 smithi132 ceph-mon[23807]: osdmap e94: 8 total, 8 up, 8 in 2024-05-09T03:50:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:33 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2942491637' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3364333504"}]: dispatch 2024-05-09T03:50:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:33 smithi078 ceph-mon[22905]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:33 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/147802961"}]': finished 2024-05-09T03:50:34.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:33 smithi078 ceph-mon[22905]: osdmap e94: 8 total, 8 up, 8 in 2024-05-09T03:50:34.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:33 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2942491637' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3364333504"}]: dispatch 2024-05-09T03:50:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:33 smithi078 ceph-mon[27629]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:33 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/147802961"}]': finished 2024-05-09T03:50:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:33 smithi078 ceph-mon[27629]: osdmap e94: 8 total, 8 up, 8 in 2024-05-09T03:50:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:33 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2942491637' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3364333504"}]: dispatch 2024-05-09T03:50:35.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:34 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2942491637' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3364333504"}]': finished 2024-05-09T03:50:35.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:34 smithi132 ceph-mon[23807]: osdmap e95: 8 total, 8 up, 8 in 2024-05-09T03:50:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:34 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2942491637' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3364333504"}]': finished 2024-05-09T03:50:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:34 smithi078 ceph-mon[22905]: osdmap e95: 8 total, 8 up, 8 in 2024-05-09T03:50:35.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:34 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2942491637' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3364333504"}]': finished 2024-05-09T03:50:35.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:34 smithi078 ceph-mon[27629]: osdmap e95: 8 total, 8 up, 8 in 2024-05-09T03:50:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:35 smithi132 ceph-mon[23807]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:36.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:35 smithi078 ceph-mon[22905]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:35 smithi078 ceph-mon[27629]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:37 smithi132 ceph-mon[23807]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:50:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:37 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:50:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:37 smithi078 ceph-mon[22905]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:50:38.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:37 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:50:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:37 smithi078 ceph-mon[27629]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:50:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:37 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:50:40.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:39 smithi132 ceph-mon[23807]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-05-09T03:50:40.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:39 smithi078 ceph-mon[22905]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-05-09T03:50:40.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:39 smithi078 ceph-mon[27629]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-05-09T03:50:42.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:41 smithi132 ceph-mon[23807]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:42.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:41 smithi078 ceph-mon[22905]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:42.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:41 smithi078 ceph-mon[27629]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:43.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:50:42 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:50:42] "GET /metrics HTTP/1.1" 200 208159 "" "Prometheus/2.33.4" 2024-05-09T03:50:43.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:50:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:50:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:50:44.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:43 smithi132 ceph-mon[23807]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-05-09T03:50:44.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:43 smithi078 ceph-mon[22905]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-05-09T03:50:44.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:43 smithi078 ceph-mon[27629]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-05-09T03:50:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:45 smithi132 ceph-mon[23807]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 932 B/s rd, 0 op/s 2024-05-09T03:50:46.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:45 smithi078 ceph-mon[22905]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 932 B/s rd, 0 op/s 2024-05-09T03:50:46.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:45 smithi078 ceph-mon[27629]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 932 B/s rd, 0 op/s 2024-05-09T03:50:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:47 smithi132 ceph-mon[23807]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:47 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:50:48.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:47 smithi078 ceph-mon[22905]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:48.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:47 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:50:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:47 smithi078 ceph-mon[27629]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:47 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:50:50.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:49 smithi132 ceph-mon[23807]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:50:50.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:49 smithi078 ceph-mon[27629]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:50:50.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:49 smithi078 ceph-mon[22905]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:50:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:51 smithi132 ceph-mon[23807]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:52.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:51 smithi078 ceph-mon[27629]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:52.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:51 smithi078 ceph-mon[22905]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:53.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:50:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:50:52] "GET /metrics HTTP/1.1" 200 208137 "" "Prometheus/2.33.4" 2024-05-09T03:50:53.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:50:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:50:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:50:54.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:53 smithi132 ceph-mon[23807]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:50:54.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:53 smithi078 ceph-mon[22905]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:50:54.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:53 smithi078 ceph-mon[27629]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:50:56.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:55 smithi132 ceph-mon[23807]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:50:56.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:55 smithi078 ceph-mon[27629]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:50:56.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:55 smithi078 ceph-mon[22905]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:50:58.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:57 smithi132 ceph-mon[23807]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:58.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:57 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:50:58.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:57 smithi078 ceph-mon[22905]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:58.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:57 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:50:58.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:57 smithi078 ceph-mon[27629]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:50:58.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:57 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:00.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:50:59 smithi132 ceph-mon[23807]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:00.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:50:59 smithi078 ceph-mon[22905]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:00.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:50:59 smithi078 ceph-mon[27629]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:02.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:01 smithi132 ceph-mon[23807]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:02.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:01 smithi078 ceph-mon[22905]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:02.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:01 smithi078 ceph-mon[27629]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:03.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:51:02 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:51:02] "GET /metrics HTTP/1.1" 200 208089 "" "Prometheus/2.33.4" 2024-05-09T03:51:03.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:51:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:51:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:51:04.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:03 smithi132 ceph-mon[23807]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:04.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:03 smithi078 ceph-mon[22905]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:04.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:03 smithi078 ceph-mon[27629]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:06.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:05 smithi132 ceph-mon[23807]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:06.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:05 smithi078 ceph-mon[22905]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:06.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:05 smithi078 ceph-mon[27629]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:08.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:07 smithi132 ceph-mon[23807]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:08.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:07 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:08.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:07 smithi078 ceph-mon[27629]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:08.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:07 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:08.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:07 smithi078 ceph-mon[22905]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:08.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:07 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:10.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:09 smithi132 ceph-mon[23807]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:10.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:09 smithi078 ceph-mon[27629]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:10.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:09 smithi078 ceph-mon[22905]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:12.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:11 smithi132 ceph-mon[23807]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:12.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:11 smithi078 ceph-mon[22905]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:12.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:11 smithi078 ceph-mon[27629]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:13.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:51:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:51:12] "GET /metrics HTTP/1.1" 200 208089 "" "Prometheus/2.33.4" 2024-05-09T03:51:13.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:51:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:51:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:51:14.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:14 smithi132 ceph-mon[23807]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:14.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:14 smithi078 ceph-mon[22905]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:14.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:14 smithi078 ceph-mon[27629]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:15.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:51:15.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:51:15.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:51:15.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:51:15.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:51:15.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:51:15.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:51:15.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:51:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:51:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:51:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:51:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:51:16.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:16 smithi078 ceph-mon[22905]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:16.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:16 smithi078 ceph-mon[27629]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:16 smithi132 ceph-mon[23807]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:18.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:18 smithi132 ceph-mon[23807]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:18.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:18 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:18.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:18 smithi078 ceph-mon[22905]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:18.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:18 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:18.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:18 smithi078 ceph-mon[27629]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:18.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:18 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:20.501 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:20 smithi078 ceph-mon[27629]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:20.502 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:20 smithi078 ceph-mon[22905]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:20.502 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:20 smithi132 ceph-mon[23807]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:21 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:51:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:21 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:51:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:21 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:51:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:21 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:51:21.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:21 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:51:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:21 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:51:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:21 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:51:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:21 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:51:21.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:21 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:51:21.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:21 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:51:21.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:21 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:51:21.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:21 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:51:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:22 smithi132 ceph-mon[23807]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:22.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:22 smithi078 ceph-mon[22905]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:22.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:22 smithi078 ceph-mon[27629]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:23.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:51:22 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:51:22] "GET /metrics HTTP/1.1" 200 208054 "" "Prometheus/2.33.4" 2024-05-09T03:51:23.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:51:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:51:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:51:24.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:24 smithi132 ceph-mon[23807]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:24.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:24 smithi078 ceph-mon[22905]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:24.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:24 smithi078 ceph-mon[27629]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:26.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:26 smithi132 ceph-mon[23807]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:26.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:26 smithi078 ceph-mon[22905]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:26.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:26 smithi078 ceph-mon[27629]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:28 smithi132 ceph-mon[23807]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:28 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:28 smithi078 ceph-mon[22905]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:28.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:28 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:28 smithi078 ceph-mon[27629]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:28 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:30.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:30 smithi132 ceph-mon[23807]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:30.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:30 smithi078 ceph-mon[22905]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:30.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:30 smithi078 ceph-mon[27629]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:32.306 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:32 smithi132 ceph-mon[23807]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:32.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:32 smithi078 ceph-mon[22905]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:32 smithi078 ceph-mon[27629]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:33.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:51:32 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:51:32] "GET /metrics HTTP/1.1" 200 208062 "" "Prometheus/2.33.4" 2024-05-09T03:51:33.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:51:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:51:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:51:34.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:34 smithi132 ceph-mon[23807]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:34.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:34 smithi078 ceph-mon[22905]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:34.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:34 smithi078 ceph-mon[27629]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:36 smithi132 ceph-mon[23807]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:36.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:36 smithi078 ceph-mon[22905]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:36.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:36 smithi078 ceph-mon[27629]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:38 smithi132 ceph-mon[23807]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:38 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:38 smithi078 ceph-mon[22905]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:38 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:38.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:38 smithi078 ceph-mon[27629]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:38.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:38 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:40.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:40 smithi132 ceph-mon[23807]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:40.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:40 smithi078 ceph-mon[22905]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:40.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:40 smithi078 ceph-mon[27629]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:42.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:42 smithi132 ceph-mon[23807]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:42.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:42 smithi078 ceph-mon[22905]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:42.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:42 smithi078 ceph-mon[27629]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:43.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:51:42 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:51:42] "GET /metrics HTTP/1.1" 200 208062 "" "Prometheus/2.33.4" 2024-05-09T03:51:43.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:51:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:51:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:51:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:44 smithi132 ceph-mon[23807]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:44.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:44 smithi078 ceph-mon[22905]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:44 smithi078 ceph-mon[27629]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:46.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:46 smithi132 ceph-mon[23807]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:46.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:46 smithi078 ceph-mon[22905]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:46 smithi078 ceph-mon[27629]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:48 smithi132 ceph-mon[23807]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:48 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:48.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:48 smithi078 ceph-mon[27629]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:48.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:48 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:48 smithi078 ceph-mon[22905]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:48 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:50 smithi132 ceph-mon[23807]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:50.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:50 smithi078 ceph-mon[22905]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:50.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:50 smithi078 ceph-mon[27629]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:52 smithi132 ceph-mon[23807]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:52.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:52 smithi078 ceph-mon[27629]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:52.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:52 smithi078 ceph-mon[22905]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:53.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:51:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:51:52] "GET /metrics HTTP/1.1" 200 208061 "" "Prometheus/2.33.4" 2024-05-09T03:51:53.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:51:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:51:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:51:54.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:54 smithi132 ceph-mon[23807]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:54.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:54 smithi078 ceph-mon[22905]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:54.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:54 smithi078 ceph-mon[27629]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:56.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:56 smithi132 ceph-mon[23807]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:56.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:56 smithi078 ceph-mon[22905]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:56.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:56 smithi078 ceph-mon[27629]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:51:58.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:58 smithi132 ceph-mon[23807]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:58.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:51:58 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:58 smithi078 ceph-mon[22905]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:51:58 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:51:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:58 smithi078 ceph-mon[27629]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:51:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:51:58 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:00 smithi132 ceph-mon[23807]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:00.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:00 smithi078 ceph-mon[22905]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:00.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:00 smithi078 ceph-mon[27629]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:02.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:02 smithi132 ceph-mon[23807]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:02.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:02 smithi078 ceph-mon[27629]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:02.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:02 smithi078 ceph-mon[22905]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:03.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:52:02 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:52:02] "GET /metrics HTTP/1.1" 200 208057 "" "Prometheus/2.33.4" 2024-05-09T03:52:03.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:52:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:52:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:52:04.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:04 smithi132 ceph-mon[23807]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:04.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:04 smithi078 ceph-mon[22905]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:04.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:04 smithi078 ceph-mon[27629]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:06.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:06 smithi132 ceph-mon[23807]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:06.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:06 smithi078 ceph-mon[22905]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:06 smithi078 ceph-mon[27629]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:08.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:08 smithi132 ceph-mon[23807]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:08.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:08 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:08.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:08 smithi078 ceph-mon[22905]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:08.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:08 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:08.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:08 smithi078 ceph-mon[27629]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:08.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:08 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:10.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:10 smithi132 ceph-mon[23807]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:10.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:10 smithi078 ceph-mon[22905]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:10 smithi078 ceph-mon[27629]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:12 smithi132 ceph-mon[23807]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:12.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:12 smithi078 ceph-mon[22905]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:12.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:12 smithi078 ceph-mon[27629]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:13.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:52:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:52:12] "GET /metrics HTTP/1.1" 200 208057 "" "Prometheus/2.33.4" 2024-05-09T03:52:13.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:52:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:52:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:52:14.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:14 smithi132 ceph-mon[23807]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:14.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:14 smithi078 ceph-mon[22905]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:14.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:14 smithi078 ceph-mon[27629]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:52:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:52:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:52:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:52:15.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:52:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:52:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:52:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:52:15.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:52:15.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:52:15.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:52:15.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:52:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:16 smithi132 ceph-mon[23807]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:16 smithi078 ceph-mon[22905]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:16.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:16 smithi078 ceph-mon[27629]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:18.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:18 smithi132 ceph-mon[23807]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:18.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:18 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:18.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:18 smithi078 ceph-mon[22905]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:18.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:18 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:18.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:18 smithi078 ceph-mon[27629]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:18.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:18 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:20.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:20 smithi132 ceph-mon[23807]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:20.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:20 smithi078 ceph-mon[22905]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:20 smithi078 ceph-mon[27629]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:21.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:21 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:52:21.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:21 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:52:21.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:21 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:52:21.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:21 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:52:21.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:21 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:52:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:21 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:52:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:21 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:52:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:21 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:52:21.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:21 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:52:21.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:21 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:52:21.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:21 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:52:21.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:21 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:52:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:22 smithi132 ceph-mon[23807]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:22.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:22 smithi078 ceph-mon[22905]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:22.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:22 smithi078 ceph-mon[27629]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:23.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:52:22 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:52:22] "GET /metrics HTTP/1.1" 200 208054 "" "Prometheus/2.33.4" 2024-05-09T03:52:23.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:52:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:52:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:52:24.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:24 smithi132 ceph-mon[23807]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:24.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:24 smithi078 ceph-mon[22905]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:24.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:24 smithi078 ceph-mon[27629]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:26.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:26 smithi132 ceph-mon[23807]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:26.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:26 smithi078 ceph-mon[27629]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:26.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:26 smithi078 ceph-mon[22905]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:28 smithi132 ceph-mon[23807]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:28 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:28 smithi078 ceph-mon[22905]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:28.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:28 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:28 smithi078 ceph-mon[27629]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:28 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:30.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:30 smithi132 ceph-mon[23807]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:30.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:30 smithi078 ceph-mon[22905]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:30.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:30 smithi078 ceph-mon[27629]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:32 smithi132 ceph-mon[23807]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:32.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:32 smithi078 ceph-mon[22905]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:32 smithi078 ceph-mon[27629]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:33.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:52:32 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:52:32] "GET /metrics HTTP/1.1" 200 208040 "" "Prometheus/2.33.4" 2024-05-09T03:52:33.736 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:52:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:52:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:52:34.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:34 smithi132 ceph-mon[23807]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:34.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:34 smithi078 ceph-mon[22905]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:34.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:34 smithi078 ceph-mon[27629]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:36 smithi132 ceph-mon[23807]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:36.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:36 smithi078 ceph-mon[27629]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:36.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:36 smithi078 ceph-mon[22905]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:38 smithi132 ceph-mon[23807]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:38 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:38.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:38 smithi078 ceph-mon[27629]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:38.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:38 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:38 smithi078 ceph-mon[22905]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:38 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:40.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:40 smithi132 ceph-mon[23807]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:40.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:40 smithi078 ceph-mon[22905]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:40.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:40 smithi078 ceph-mon[27629]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:42.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:42 smithi132 ceph-mon[23807]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:42.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:42 smithi078 ceph-mon[22905]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:42.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:42 smithi078 ceph-mon[27629]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:43.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:52:42 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:52:42] "GET /metrics HTTP/1.1" 200 208040 "" "Prometheus/2.33.4" 2024-05-09T03:52:43.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:52:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:52:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:52:44.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:44 smithi132 ceph-mon[23807]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:44 smithi078 ceph-mon[22905]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:44 smithi078 ceph-mon[27629]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:46.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:46 smithi132 ceph-mon[23807]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:46.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:46 smithi078 ceph-mon[22905]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:46 smithi078 ceph-mon[27629]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:48 smithi132 ceph-mon[23807]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:48 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:48.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:48 smithi078 ceph-mon[27629]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:48.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:48 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:48 smithi078 ceph-mon[22905]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:48 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:50 smithi132 ceph-mon[23807]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:50 smithi078 ceph-mon[22905]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:50.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:50 smithi078 ceph-mon[27629]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:52 smithi132 ceph-mon[23807]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:52.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:52 smithi078 ceph-mon[22905]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:52.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:52 smithi078 ceph-mon[27629]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:53.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:52:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:52:52] "GET /metrics HTTP/1.1" 200 208006 "" "Prometheus/2.33.4" 2024-05-09T03:52:53.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:52:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:52:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:52:54.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:54 smithi132 ceph-mon[23807]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:54.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:54 smithi078 ceph-mon[22905]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:54.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:54 smithi078 ceph-mon[27629]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:56.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:56 smithi132 ceph-mon[23807]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:56.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:56 smithi078 ceph-mon[27629]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:56.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:56 smithi078 ceph-mon[22905]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:52:58.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:58 smithi132 ceph-mon[23807]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:58.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:52:58 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:58.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:58 smithi078 ceph-mon[27629]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:52:58 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:52:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:58 smithi078 ceph-mon[22905]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:52:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:52:58 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:00 smithi132 ceph-mon[23807]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:00 smithi078 ceph-mon[22905]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:00.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:00 smithi078 ceph-mon[27629]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:02.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:02 smithi132 ceph-mon[23807]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:02.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:02 smithi078 ceph-mon[22905]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:02 smithi078 ceph-mon[27629]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:03.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:53:02 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:53:02] "GET /metrics HTTP/1.1" 200 208001 "" "Prometheus/2.33.4" 2024-05-09T03:53:03.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:53:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:53:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:53:04.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:04 smithi132 ceph-mon[23807]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:04.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:04 smithi078 ceph-mon[22905]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:04.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:04 smithi078 ceph-mon[27629]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:06.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:06 smithi132 ceph-mon[23807]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:06.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:06 smithi078 ceph-mon[22905]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:06 smithi078 ceph-mon[27629]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:08.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:08 smithi132 ceph-mon[23807]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:08.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:08 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:08.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:08 smithi078 ceph-mon[22905]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:08.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:08 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:08.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:08 smithi078 ceph-mon[27629]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:08.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:08 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:09.526 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-05-09T03:53:10.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:10 smithi132 ceph-mon[23807]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:10 smithi078 ceph-mon[22905]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:10 smithi078 ceph-mon[27629]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:12.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-05-09T03:53:12.483 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:12 smithi078 ceph-mon[27629]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:12.483 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:12 smithi078 ceph-mon[22905]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:12 smithi132 ceph-mon[23807]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:12.762 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:53:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:53:12] "GET /metrics HTTP/1.1" 200 208001 "" "Prometheus/2.33.4" 2024-05-09T03:53:13.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:53:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:53:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:53:14.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:14 smithi132 ceph-mon[23807]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:14.506 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:14 smithi078 ceph-mon[27629]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:14.506 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:14 smithi078 ceph-mon[22905]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:14.867 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph config set global log_to_journald false --force' 2024-05-09T03:53:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:53:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:53:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:53:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:53:15.509 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:53:15.509 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:53:15.509 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:53:15.509 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:53:15.509 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:53:15.510 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:53:15.510 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:53:15.510 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:53:16.443 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:16 smithi078 ceph-mon[27629]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:16.443 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:16 smithi078 ceph-mon[22905]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:16 smithi132 ceph-mon[23807]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:17.444 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T03:53:18.418 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:18 smithi078 ceph-mon[22905]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:18.418 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:18 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:18.418 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:18 smithi078 ceph-mon[27629]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:18.418 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:18 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:18.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:18 smithi132 ceph-mon[23807]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:18.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:18 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:19.441 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T03:53:19.441 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (3m) 2m ago 3m 13.7M - ba2b418f427c 1bc0d1c4e0c8 2024-05-09T03:53:19.441 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (3m) 2m ago 3m 47.2M - 8.3.5 dad864ee21e9 c1b7ab086083 2024-05-09T03:53:19.442 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (3m) 2m ago 3m 71.1M - 3.5 e1d6a67b021e 65c58273a7db 2024-05-09T03:53:19.442 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443 running (8m) 2m ago 8m 410M - 17.2.0 e1d6a67b021e c865a878af2f 2024-05-09T03:53:19.442 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:9283 running (10m) 2m ago 10m 454M - 17.2.0 e1d6a67b021e b2a9d11f22f2 2024-05-09T03:53:19.442 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (10m) 2m ago 10m 46.6M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T03:53:19.443 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (8m) 2m ago 8m 41.9M 2048M 17.2.0 e1d6a67b021e a71bae52559d 2024-05-09T03:53:19.443 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (8m) 2m ago 8m 42.8M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T03:53:19.443 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (4m) 2m ago 4m 16.9M - 1dbe0e931976 10d0785d9bf4 2024-05-09T03:53:19.443 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (4m) 2m ago 4m 15.6M - 1dbe0e931976 dfbd85c54dbd 2024-05-09T03:53:19.443 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 2m ago 8m 48.1M 4016M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T03:53:19.443 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 2m ago 7m 51.2M 4016M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T03:53:19.444 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (7m) 2m ago 7m 44.7M 4016M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T03:53:19.444 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (7m) 2m ago 7m 45.8M 4016M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T03:53:19.444 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (6m) 2m ago 6m 47.8M 4272M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T03:53:19.444 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (6m) 2m ago 6m 44.6M 4272M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T03:53:19.444 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (5m) 2m ago 5m 44.8M 4272M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T03:53:19.444 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (5m) 2m ago 5m 48.3M 4272M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T03:53:19.444 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (3m) 2m ago 4m 43.1M - 514e6a882f6e 69b608c5f2ac 2024-05-09T03:53:19.444 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (3m) 2m ago 3m 70.9M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T03:53:19.444 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (3m) 2m ago 3m 72.1M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T03:53:20.118 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions' 2024-05-09T03:53:20.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:20 smithi132 ceph-mon[23807]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:20.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:20 smithi132 ceph-mon[23807]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:53:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:20 smithi078 ceph-mon[22905]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:20 smithi078 ceph-mon[22905]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:53:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:20 smithi078 ceph-mon[27629]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:20 smithi078 ceph-mon[27629]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:53:21.398 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:21 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:53:21.398 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:21 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:53:21.398 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:21 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:53:21.398 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:21 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:53:21.398 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:21 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:53:21.398 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:21 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:53:21.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:21 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:53:21.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:21 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:53:21.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:21 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:53:22.128 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T03:53:22.128 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T03:53:22.128 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-09T03:53:22.128 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:53:22.128 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T03:53:22.128 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T03:53:22.128 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:53:22.128 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T03:53:22.129 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T03:53:22.129 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:53:22.129 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": {}, 2024-05-09T03:53:22.129 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T03:53:22.129 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T03:53:22.129 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:53:22.129 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T03:53:22.129 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-05-09T03:53:22.129 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T03:53:22.129 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T03:53:22.663 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:22 smithi078 ceph-mon[27629]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:22.663 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:22 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:53:22.663 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:22 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2541611153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T03:53:22.663 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:22 smithi078 ceph-mon[22905]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:22.663 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:22 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:53:22.663 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:22 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2541611153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T03:53:22.686 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph -s' 2024-05-09T03:53:22.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:22 smithi132 ceph-mon[23807]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:22.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:22 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:53:22.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:22 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2541611153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T03:53:23.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:53:22 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:53:22] "GET /metrics HTTP/1.1" 200 208002 "" "Prometheus/2.33.4" 2024-05-09T03:53:23.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:53:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:53:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:53:24.668 INFO:teuthology.orchestra.run.smithi078.stdout: cluster: 2024-05-09T03:53:24.669 INFO:teuthology.orchestra.run.smithi078.stdout: id: 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:53:24.669 INFO:teuthology.orchestra.run.smithi078.stdout: health: HEALTH_OK 2024-05-09T03:53:24.669 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:53:24.669 INFO:teuthology.orchestra.run.smithi078.stdout: services: 2024-05-09T03:53:24.669 INFO:teuthology.orchestra.run.smithi078.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-05-09T03:53:24.669 INFO:teuthology.orchestra.run.smithi078.stdout: mgr: y(active, since 5m), standbys: x 2024-05-09T03:53:24.669 INFO:teuthology.orchestra.run.smithi078.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-05-09T03:53:24.669 INFO:teuthology.orchestra.run.smithi078.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-09T03:53:24.669 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:53:24.669 INFO:teuthology.orchestra.run.smithi078.stdout: data: 2024-05-09T03:53:24.669 INFO:teuthology.orchestra.run.smithi078.stdout: pools: 6 pools, 161 pgs 2024-05-09T03:53:24.670 INFO:teuthology.orchestra.run.smithi078.stdout: objects: 241 objects, 585 KiB 2024-05-09T03:53:24.670 INFO:teuthology.orchestra.run.smithi078.stdout: usage: 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:53:24.670 INFO:teuthology.orchestra.run.smithi078.stdout: pgs: 161 active+clean 2024-05-09T03:53:24.670 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:53:24.670 INFO:teuthology.orchestra.run.smithi078.stdout: io: 2024-05-09T03:53:24.670 INFO:teuthology.orchestra.run.smithi078.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-09T03:53:24.670 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:53:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:24 smithi078 ceph-mon[22905]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:24 smithi078 ceph-mon[27629]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:25.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:24 smithi132 ceph-mon[23807]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:25.333 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ls' 2024-05-09T03:53:25.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:25 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1430548569' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T03:53:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:25 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1430548569' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T03:53:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:25 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1430548569' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T03:53:26.750 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:26 smithi078 ceph-mon[27629]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:26.750 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:26 smithi078 ceph-mon[22905]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:27.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:26 smithi132 ceph-mon[23807]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:27.430 INFO:teuthology.orchestra.run.smithi078.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-05-09T03:53:27.430 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi078=a;count:1 2024-05-09T03:53:27.430 INFO:teuthology.orchestra.run.smithi078.stdout:grafana ?:3000 1/1 3m ago 5m smithi132=a;count:1 2024-05-09T03:53:27.431 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-05-09T03:53:27.431 INFO:teuthology.orchestra.run.smithi078.stdout:mgr 2/2 3m ago 8m smithi078=y;smithi132=x;count:2 2024-05-09T03:53:27.431 INFO:teuthology.orchestra.run.smithi078.stdout:mon 3/3 3m ago 9m smithi078:172.21.15.78=a;smithi078:[v2:172.21.15.78:3301,v1:172.21.15.78:6790]=c;smithi132:172.21.15.132=b;count:3 2024-05-09T03:53:27.431 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi078=a;smithi132=b;count:2 2024-05-09T03:53:27.431 INFO:teuthology.orchestra.run.smithi078.stdout:osd 8 3m ago - 2024-05-09T03:53:27.431 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus ?:9095 1/1 3m ago 5m smithi132=a;count:1 2024-05-09T03:53:27.431 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-05-09T03:53:28.231 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-09T03:53:28.851 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:28 smithi078 ceph-mon[27629]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:28.852 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:28 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:28.852 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:28 smithi078 ceph-mon[27629]: from='client.24883 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:53:28.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:28 smithi078 ceph-mon[22905]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:28.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:28 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:28.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:28 smithi078 ceph-mon[22905]: from='client.24883 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:53:29.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:28 smithi132 ceph-mon[23807]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:29.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:28 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:29.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:28 smithi132 ceph-mon[23807]: from='client.24883 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:53:30.476 INFO:teuthology.orchestra.run.smithi078.stdout:Scheduled to redeploy mgr.x on host 'smithi132' 2024-05-09T03:53:30.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:30 smithi078 ceph-mon[22905]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:30.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:30 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3510739755' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-09T03:53:30.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:30 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:53:30.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:30 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:53:30.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:30 smithi078 ceph-mon[27629]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:30.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:30 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3510739755' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-09T03:53:30.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:30 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:53:30.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:30 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:53:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:30 smithi132 ceph-mon[23807]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:30 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3510739755' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-09T03:53:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:30 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:53:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:30 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:53:31.297 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps --refresh' 2024-05-09T03:53:31.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:31 smithi078 ceph-mon[27629]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:53:31.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:31 smithi078 ceph-mon[27629]: Schedule redeploy daemon mgr.x 2024-05-09T03:53:31.784 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:31 smithi078 ceph-mon[22905]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:53:31.784 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:31 smithi078 ceph-mon[22905]: Schedule redeploy daemon mgr.x 2024-05-09T03:53:32.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:31 smithi132 ceph-mon[23807]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:53:32.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:31 smithi132 ceph-mon[23807]: Schedule redeploy daemon mgr.x 2024-05-09T03:53:32.789 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:32 smithi078 ceph-mon[22905]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:32.789 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:32 smithi078 ceph-mon[27629]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:32.790 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:53:32 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:53:32] "GET /metrics HTTP/1.1" 200 207988 "" "Prometheus/2.33.4" 2024-05-09T03:53:32.904 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:32 smithi132 ceph-mon[23807]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:33.389 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T03:53:33.389 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (3m) 3m ago 4m 13.7M - ba2b418f427c 1bc0d1c4e0c8 2024-05-09T03:53:33.389 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (3m) 3m ago 3m 47.2M - 8.3.5 dad864ee21e9 c1b7ab086083 2024-05-09T03:53:33.389 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (3m) 3m ago 3m 71.1M - 3.5 e1d6a67b021e 65c58273a7db 2024-05-09T03:53:33.389 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e c865a878af2f 2024-05-09T03:53:33.389 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e b2a9d11f22f2 2024-05-09T03:53:33.389 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (10m) 3m ago 10m 46.6M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T03:53:33.389 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (9m) 3m ago 9m 41.9M 2048M 17.2.0 e1d6a67b021e a71bae52559d 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (9m) 3m ago 9m 42.8M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (4m) 3m ago 4m 16.9M - 1dbe0e931976 10d0785d9bf4 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (4m) 3m ago 4m 15.6M - 1dbe0e931976 dfbd85c54dbd 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 3m ago 8m 48.1M 4016M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (8m) 3m ago 8m 51.2M 4016M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (7m) 3m ago 7m 44.7M 4016M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (7m) 3m ago 7m 45.8M 4016M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (6m) 3m ago 6m 47.8M 4272M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (6m) 3m ago 6m 44.6M 4272M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (6m) 3m ago 6m 44.8M 4272M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (5m) 3m ago 5m 48.3M 4272M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (3m) 3m ago 5m 43.1M - 514e6a882f6e 69b608c5f2ac 2024-05-09T03:53:33.390 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (3m) 3m ago 3m 70.9M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T03:53:33.391 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (3m) 3m ago 3m 72.1M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T03:53:33.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:53:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:53:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:53:34.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'sleep 180' 2024-05-09T03:53:34.759 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:34 smithi078 ceph-mon[22905]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:34.759 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:34 smithi078 ceph-mon[22905]: from='client.15003 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:53:34.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:34 smithi078 ceph-mon[27629]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:34.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:34 smithi078 ceph-mon[27629]: from='client.15003 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:53:35.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:34 smithi132 ceph-mon[23807]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:35.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:34 smithi132 ceph-mon[23807]: from='client.15003 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:53:36.978 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:36 smithi078 ceph-mon[22905]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:36.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:36 smithi078 ceph-mon[27629]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:36 smithi132 ceph-mon[23807]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:37 smithi132 ceph-mon[23807]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:37 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:38.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:37 smithi078 ceph-mon[22905]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:38.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:37 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:37 smithi078 ceph-mon[27629]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:37 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:40.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:39 smithi132 ceph-mon[23807]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:40.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:39 smithi078 ceph-mon[22905]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:40.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:39 smithi078 ceph-mon[27629]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:42.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:41 smithi132 ceph-mon[23807]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:42.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:41 smithi078 ceph-mon[22905]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:42.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:41 smithi078 ceph-mon[27629]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:43.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:53:42 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:53:42] "GET /metrics HTTP/1.1" 200 207988 "" "Prometheus/2.33.4" 2024-05-09T03:53:43.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:53:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:53:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:53:44.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:43 smithi132 ceph-mon[23807]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:44.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:43 smithi078 ceph-mon[27629]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:44.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:43 smithi078 ceph-mon[22905]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:45 smithi132 ceph-mon[23807]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:46.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:45 smithi078 ceph-mon[22905]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:46.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:46 smithi078 ceph-mon[27629]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:48.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:48 smithi078 ceph-mon[22905]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:48.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:48 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:48 smithi078 ceph-mon[27629]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:48 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:48 smithi132 ceph-mon[23807]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:48 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:50.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:50 smithi078 ceph-mon[22905]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:50.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:50 smithi078 ceph-mon[27629]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:50 smithi132 ceph-mon[23807]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:52.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:52 smithi078 ceph-mon[22905]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:52.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:52 smithi078 ceph-mon[27629]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:52 smithi132 ceph-mon[23807]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:53.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:53:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:53:52] "GET /metrics HTTP/1.1" 200 207994 "" "Prometheus/2.33.4" 2024-05-09T03:53:53.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:53:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:53:54.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:54 smithi132 ceph-mon[23807]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:54.515 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:54 smithi078 ceph-mon[22905]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:54.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:54 smithi078 ceph-mon[27629]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:56.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:56 smithi132 ceph-mon[23807]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:56.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:56 smithi078 ceph-mon[22905]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:56.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:56 smithi078 ceph-mon[27629]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:53:58.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:58 smithi132 ceph-mon[23807]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:58.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:53:58 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:58 smithi078 ceph-mon[22905]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:53:58 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:53:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:58 smithi078 ceph-mon[27629]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:53:58.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:53:58 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:00 smithi132 ceph-mon[23807]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:00 smithi078 ceph-mon[22905]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:00.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:00 smithi078 ceph-mon[27629]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:02.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:02 smithi078 ceph-mon[22905]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:02.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:02 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:02.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:02 smithi132 ceph-mon[23807]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:02.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:02 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:02.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:02 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:54:02.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:02 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:54:02.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:02 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:54:02.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:02 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:54:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:02 smithi078 ceph-mon[27629]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:02 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:02.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:02 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:54:02.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:02 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:54:03.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:54:02 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:54:02] "GET /metrics HTTP/1.1" 200 208008 "" "Prometheus/2.33.4" 2024-05-09T03:54:03.383 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:03 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:03.496 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:03 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:03.496 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:03 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:03.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:54:04.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:04 smithi132 ceph-mon[23807]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:04.370 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:04 smithi078 ceph-mon[22905]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:04.370 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:04 smithi078 ceph-mon[27629]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:06.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:06 smithi132 ceph-mon[23807]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:06.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:06 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:06.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:06 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:06.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:06 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:06.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:06 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T03:54:06.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:06 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T03:54:06.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:06 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T03:54:06.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:06 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:54:06.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:06 smithi132 ceph-mon[23807]: Deploying daemon mgr.x on smithi132 2024-05-09T03:54:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[27629]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T03:54:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T03:54:06.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T03:54:06.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:54:06.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[27629]: Deploying daemon mgr.x on smithi132 2024-05-09T03:54:06.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[22905]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:06.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:06.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:06.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:06.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T03:54:06.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T03:54:06.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T03:54:06.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:54:06.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:06 smithi078 ceph-mon[22905]: Deploying daemon mgr.x on smithi132 2024-05-09T03:54:08.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:08 smithi132 ceph-mon[23807]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:08.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:08 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:08.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:08 smithi078 ceph-mon[22905]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:08.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:08 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:08.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:08 smithi078 ceph-mon[27629]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:08.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:08 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:10.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:10 smithi132 ceph-mon[23807]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:10 smithi078 ceph-mon[22905]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:10.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:10 smithi078 ceph-mon[27629]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:12 smithi132 ceph-mon[23807]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:12.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:12 smithi078 ceph-mon[22905]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:12.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:12 smithi078 ceph-mon[27629]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:13.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:54:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:54:12] "GET /metrics HTTP/1.1" 200 208008 "" "Prometheus/2.33.4" 2024-05-09T03:54:13.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:54:14.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:54:13.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:54:14.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:54:13.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:54:14.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:14 smithi132 ceph-mon[23807]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:14.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:14 smithi078 ceph-mon[22905]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:14.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:14 smithi078 ceph-mon[27629]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:54:15.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:54:15.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:54:15.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:54:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:54:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:54:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:54:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:54:15.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:54:15.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:54:15.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:54:15.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:54:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:16 smithi132 ceph-mon[23807]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:16 smithi078 ceph-mon[22905]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:16.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:16 smithi078 ceph-mon[27629]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:18.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:18 smithi132 ceph-mon[23807]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:18.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:18 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:18.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:18 smithi078 ceph-mon[22905]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:18.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:18 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:18.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:18 smithi078 ceph-mon[27629]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:18.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:18 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:20.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:20 smithi132 ceph-mon[23807]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:20 smithi078 ceph-mon[27629]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:20 smithi078 ceph-mon[22905]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:22 smithi132 ceph-mon[23807]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:22.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:22 smithi078 ceph-mon[22905]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:22.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:22 smithi078 ceph-mon[27629]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:23.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:54:22 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:54:22] "GET /metrics HTTP/1.1" 200 208020 "" "Prometheus/2.33.4" 2024-05-09T03:54:23.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:54:23.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:54:23.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:54:23.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:54:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:54:23.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:54:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:54:24.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:24 smithi132 ceph-mon[23807]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:24.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:24 smithi078 ceph-mon[22905]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:24.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:24 smithi078 ceph-mon[27629]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:26.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:26 smithi132 ceph-mon[23807]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:26.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:26 smithi078 ceph-mon[22905]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:26.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:26 smithi078 ceph-mon[27629]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:28 smithi132 ceph-mon[23807]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:28 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:28.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:28 smithi078 ceph-mon[22905]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:28.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:28 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:28 smithi078 ceph-mon[27629]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:28 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:30.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:30 smithi132 ceph-mon[23807]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:30 smithi078 ceph-mon[22905]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:30.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:30 smithi078 ceph-mon[27629]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:32.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:32 smithi132 ceph-mon[23807]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:32.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:32 smithi078 ceph-mon[22905]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:32 smithi078 ceph-mon[27629]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:33.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:54:32 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:54:32] "GET /metrics HTTP/1.1" 200 208020 "" "Prometheus/2.33.4" 2024-05-09T03:54:33.652 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[24989]: ::ffff:172.21.15.132 - - [09/May/2024:03:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:54:33.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:54:33.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:54:33.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:54:33.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:54:33.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:54:33.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:54:34.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:34 smithi132 ceph-mon[23807]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:34.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:34 smithi078 ceph-mon[22905]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:34.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:34 smithi078 ceph-mon[27629]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:36 smithi132 ceph-mon[23807]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:36.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:36 smithi078 ceph-mon[22905]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:36.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:36 smithi078 ceph-mon[27629]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:37.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:36 smithi132 systemd[1]: Stopping Ceph mgr.x for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:54:37.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:36 smithi132 bash[43844]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr.x" found: no such container 2024-05-09T03:54:37.383 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:37 smithi132 podman[43858]: 2024-05-09 03:54:37.09384896 +0000 UTC m=+0.144810547 container died c865a878af2fd9e91ce931a5da9dec6203e9977746776dbb6d4d73dcd396e4d9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, io.openshift.expose-services=, version=8, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, architecture=x86_64, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , distribution-scope=public, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-09T03:54:37.513 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:37 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:54:37.147Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=5 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": dial tcp 172.21.15.132:8443: connect: connection refused" 2024-05-09T03:54:37.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:37 smithi132 podman[43858]: 2024-05-09 03:54:37.582015956 +0000 UTC m=+0.632977532 container cleanup c865a878af2fd9e91ce931a5da9dec6203e9977746776dbb6d4d73dcd396e4d9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, version=8, release=754, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, GIT_BRANCH=HEAD, distribution-scope=public) 2024-05-09T03:54:37.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:37 smithi132 bash[43858]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x 2024-05-09T03:54:37.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:37 smithi132 bash[43885]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr.x" found: no such container 2024-05-09T03:54:38.215 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:37 smithi132 podman[43872]: 2024-05-09 03:54:37.850356895 +0000 UTC m=+0.751720462 container remove c865a878af2fd9e91ce931a5da9dec6203e9977746776dbb6d4d73dcd396e4d9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, io.openshift.expose-services=, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , ceph=True, vendor=Red Hat, Inc., version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, name=centos-stream, vcs-type=git, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-09T03:54:38.216 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:37 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-05-09T03:54:38.216 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:37 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-05-09T03:54:38.216 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:37 smithi132 systemd[1]: Stopped Ceph mgr.x for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:54:38.216 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:37 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.x.service: Consumed 17.582s CPU time. 2024-05-09T03:54:38.216 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:38 smithi132 systemd[1]: Starting Ceph mgr.x for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:54:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:38 smithi132 ceph-mon[23807]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:38 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:38.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:38 smithi132 podman[43993]: 2024-05-09 03:54:38.164363352 +0000 UTC m=+0.018318088 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T03:54:38.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:38 smithi132 podman[43993]: 2024-05-09 03:54:38.269141288 +0000 UTC m=+0.123096052 container create 4a3f6038cf259a266e375524efe780b91398de440f8bb466cc29814be955b51e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-05-09T03:54:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:38 smithi078 ceph-mon[22905]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:38 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:38.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:38 smithi078 ceph-mon[27629]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:38.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:38 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:38.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:38 smithi132 podman[43993]: 2024-05-09 03:54:38.504942777 +0000 UTC m=+0.358897503 container init 4a3f6038cf259a266e375524efe780b91398de440f8bb466cc29814be955b51e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-05-09T03:54:38.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:38 smithi132 podman[43993]: 2024-05-09 03:54:38.507644178 +0000 UTC m=+0.361598902 container start 4a3f6038cf259a266e375524efe780b91398de440f8bb466cc29814be955b51e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5) 2024-05-09T03:54:38.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:38 smithi132 ceph-mgr[44012]: -- 172.21.15.132:0/602681040 <== mon.2 v2:172.21.15.132:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5595bda465a0 con 0x5595bda1ac00 2024-05-09T03:54:38.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:38 smithi132 bash[43993]: 4a3f6038cf259a266e375524efe780b91398de440f8bb466cc29814be955b51e 2024-05-09T03:54:38.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:38 smithi132 systemd[1]: Started Ceph mgr.x for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:54:40.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:39 smithi132 ceph-mon[23807]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:40.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:39 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:40.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:39 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:40.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:39 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:54:40.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:39 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:54:40.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:39 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:54:40.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:39 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:40.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[22905]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:40.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:40.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:40.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:54:40.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:54:40.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:54:40.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:40.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[27629]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:40.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:40.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:40.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:54:40.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:54:40.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:54:40.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:39 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:42.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:41 smithi132 ceph-mon[23807]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:42.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:41 smithi078 ceph-mon[22905]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:42.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:41 smithi078 ceph-mon[27629]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:43.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:54:42 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:54:42] "GET /metrics HTTP/1.1" 200 208020 "" "Prometheus/2.33.4" 2024-05-09T03:54:43.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:54:43.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": dial tcp 172.21.15.132:8443: connect: connection refused" 2024-05-09T03:54:43.763 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:54:43.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": dial tcp 172.21.15.132:8443: connect: connection refused" 2024-05-09T03:54:43.763 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:54:43.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:54:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:44 smithi132 ceph-mon[23807]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:44 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:44 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:44 smithi078 ceph-mon[22905]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:44 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:44 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:44 smithi078 ceph-mon[27629]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:44 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:44 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:54:46.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:46 smithi132 ceph-mon[23807]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:46.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:46 smithi132 ceph-mon[23807]: Standby manager daemon x restarted 2024-05-09T03:54:46.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:46 smithi132 ceph-mon[23807]: Standby manager daemon x started 2024-05-09T03:54:46.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:46 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T03:54:46.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:46 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T03:54:46.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:46 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T03:54:46.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:46 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T03:54:46.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:45 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:54:45] ENGINE Bus STARTING 2024-05-09T03:54:46.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:46 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: CherryPy Checker: 2024-05-09T03:54:46.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:46 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: The Application mounted at '' has an empty config. 2024-05-09T03:54:46.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:46 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T03:54:46.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:46 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:54:46] ENGINE Serving on http://:::9283 2024-05-09T03:54:46.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:46 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:54:46] ENGINE Bus STARTED 2024-05-09T03:54:46.512 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:46 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:54:46.117Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=4 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:54:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[27629]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[27629]: Standby manager daemon x restarted 2024-05-09T03:54:46.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[27629]: Standby manager daemon x started 2024-05-09T03:54:46.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T03:54:46.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T03:54:46.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T03:54:46.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T03:54:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[22905]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[22905]: Standby manager daemon x restarted 2024-05-09T03:54:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[22905]: Standby manager daemon x started 2024-05-09T03:54:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T03:54:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T03:54:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T03:54:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:46 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T03:54:47.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:47 smithi132 ceph-mon[23807]: mgrmap e21: y(active, since 6m), standbys: x 2024-05-09T03:54:47.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:47 smithi078 ceph-mon[22905]: mgrmap e21: y(active, since 6m), standbys: x 2024-05-09T03:54:47.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:47 smithi078 ceph-mon[27629]: mgrmap e21: y(active, since 6m), standbys: x 2024-05-09T03:54:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:48 smithi132 ceph-mon[23807]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:48 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:48 smithi078 ceph-mon[22905]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:48 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:48.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:48 smithi078 ceph-mon[27629]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:48.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:48 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:50 smithi132 ceph-mon[23807]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:50 smithi078 ceph-mon[22905]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:50.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:50 smithi078 ceph-mon[27629]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:52 smithi132 ceph-mon[23807]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:52.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:52 smithi078 ceph-mon[22905]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:52.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:52 smithi078 ceph-mon[27629]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:53.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:54:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:54:52] "GET /metrics HTTP/1.1" 200 208014 "" "Prometheus/2.33.4" 2024-05-09T03:54:53.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:54:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:54:53.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:54:53.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:54:53.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:54:53.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:54:53.763 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:54:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:54:53.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:54:54.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:54 smithi132 ceph-mon[23807]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:54.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:54 smithi078 ceph-mon[22905]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:54.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:54 smithi078 ceph-mon[27629]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:56.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:56 smithi132 ceph-mon[23807]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:56.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:56 smithi078 ceph-mon[22905]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:56.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:56 smithi078 ceph-mon[27629]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:54:58.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:58 smithi132 ceph-mon[23807]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:58.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:54:58 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:58 smithi078 ceph-mon[22905]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:54:58 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:54:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:58 smithi078 ceph-mon[27629]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:54:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:54:58 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:00 smithi132 ceph-mon[23807]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:00 smithi078 ceph-mon[22905]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:00.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:00 smithi078 ceph-mon[27629]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:02.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:02 smithi132 ceph-mon[23807]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:02.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:02 smithi078 ceph-mon[22905]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:02 smithi078 ceph-mon[27629]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:03.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:55:02 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:55:02] "GET /metrics HTTP/1.1" 200 208017 "" "Prometheus/2.33.4" 2024-05-09T03:55:03.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:55:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:55:03.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:55:03.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:55:03.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:55:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:55:03.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:55:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:55:04.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:04 smithi132 ceph-mon[23807]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:04.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:04 smithi078 ceph-mon[22905]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:04.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:04 smithi078 ceph-mon[27629]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:06.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:06 smithi132 ceph-mon[23807]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:06.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:06 smithi078 ceph-mon[22905]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:06 smithi078 ceph-mon[27629]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:08.413 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:08 smithi078 ceph-mon[22905]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:08.414 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:08 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:08.414 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:08 smithi078 ceph-mon[27629]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:08.414 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:08 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:08.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:08 smithi132 ceph-mon[23807]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:08.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:08 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:10.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:10 smithi132 ceph-mon[23807]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:10 smithi078 ceph-mon[22905]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:10 smithi078 ceph-mon[27629]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:12 smithi132 ceph-mon[23807]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:12.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:12 smithi078 ceph-mon[22905]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:12.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:12 smithi078 ceph-mon[27629]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:13.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:55:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:55:12] "GET /metrics HTTP/1.1" 200 208017 "" "Prometheus/2.33.4" 2024-05-09T03:55:13.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:55:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:55:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:55:13.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:55:13.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:55:13.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:55:13.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:55:13.762 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:55:13.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:55:14.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:14 smithi132 ceph-mon[23807]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:14.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:14 smithi078 ceph-mon[22905]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:14.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:14 smithi078 ceph-mon[27629]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:55:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:55:15.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:55:15.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:55:15.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:55:15.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:55:15.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:55:15.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:55:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:55:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:55:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:55:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:55:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:16 smithi132 ceph-mon[23807]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:16 smithi078 ceph-mon[22905]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:16.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:16 smithi078 ceph-mon[27629]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:18.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:18 smithi132 ceph-mon[23807]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:18.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:18 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:18.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:18 smithi078 ceph-mon[22905]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:18.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:18 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:18.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:18 smithi078 ceph-mon[27629]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:18.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:18 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:20.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:20 smithi132 ceph-mon[23807]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:20 smithi078 ceph-mon[22905]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:20 smithi078 ceph-mon[27629]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:22 smithi132 ceph-mon[23807]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:22.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:22 smithi078 ceph-mon[22905]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:22.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:22 smithi078 ceph-mon[27629]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:23.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:55:22 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:55:22] "GET /metrics HTTP/1.1" 200 208015 "" "Prometheus/2.33.4" 2024-05-09T03:55:23.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:55:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:55:24.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:55:23.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:55:24.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:55:23.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:55:24.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:55:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:55:24.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:24 smithi132 ceph-mon[23807]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:24.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:24 smithi078 ceph-mon[22905]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:24.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:24 smithi078 ceph-mon[27629]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:26.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:26 smithi132 ceph-mon[23807]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:26.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:26 smithi078 ceph-mon[22905]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:26.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:26 smithi078 ceph-mon[27629]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:28 smithi132 ceph-mon[23807]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:28 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:28.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:28 smithi078 ceph-mon[22905]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:28.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:28 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:28 smithi078 ceph-mon[27629]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:28 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:30.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:30 smithi132 ceph-mon[23807]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:30 smithi078 ceph-mon[22905]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:30.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:30 smithi078 ceph-mon[27629]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:32 smithi132 ceph-mon[23807]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:32.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:32 smithi078 ceph-mon[22905]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:32.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:32 smithi078 ceph-mon[27629]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:33.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:55:32 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:55:32] "GET /metrics HTTP/1.1" 200 208011 "" "Prometheus/2.33.4" 2024-05-09T03:55:33.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:55:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:55:34.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:55:33.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:55:34.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:55:33.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:55:34.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:55:33.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:55:34.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:34 smithi132 ceph-mon[23807]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:34.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:34 smithi078 ceph-mon[22905]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:34.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:34 smithi078 ceph-mon[27629]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:36 smithi132 ceph-mon[23807]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:36.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:36 smithi078 ceph-mon[22905]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:36.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:36 smithi078 ceph-mon[27629]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:38 smithi132 ceph-mon[23807]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:38 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:38 smithi078 ceph-mon[22905]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:38 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:38.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:38 smithi078 ceph-mon[27629]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:38.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:38 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:40.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:40 smithi132 ceph-mon[23807]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:40.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:40 smithi078 ceph-mon[22905]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:40.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:40 smithi078 ceph-mon[27629]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:42.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:42 smithi132 ceph-mon[23807]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:42 smithi078 ceph-mon[22905]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:42.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:42 smithi078 ceph-mon[27629]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:43.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:55:42 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:55:42] "GET /metrics HTTP/1.1" 200 208011 "" "Prometheus/2.33.4" 2024-05-09T03:55:43.731 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:55:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:55:43.734 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:55:43.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:55:43.734 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:55:43.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:55:43.734 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:55:43.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:55:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:44 smithi132 ceph-mon[23807]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:44 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:55:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:44 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:55:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:44 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:55:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:44 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:55:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:44 smithi078 ceph-mon[27629]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:44 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:55:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:44 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:55:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:44 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:55:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:44 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:55:44.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:44 smithi078 ceph-mon[22905]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:44.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:44 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:55:44.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:44 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:55:44.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:44 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:55:44.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:44 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:55:46.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:46 smithi132 ceph-mon[23807]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:46.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:46 smithi078 ceph-mon[22905]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:46 smithi078 ceph-mon[27629]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:48 smithi132 ceph-mon[23807]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:48 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:48 smithi078 ceph-mon[22905]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:48 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:48.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:48 smithi078 ceph-mon[27629]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:48.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:48 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:50 smithi132 ceph-mon[23807]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:50 smithi078 ceph-mon[22905]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:50.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:50 smithi078 ceph-mon[27629]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:52 smithi132 ceph-mon[23807]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:52.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:52 smithi078 ceph-mon[22905]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:52.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:52 smithi078 ceph-mon[27629]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:53.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:55:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:55:52] "GET /metrics HTTP/1.1" 200 208017 "" "Prometheus/2.33.4" 2024-05-09T03:55:53.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:55:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:55:54.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:55:53.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:55:54.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:55:53.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:55:54.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:55:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:55:53.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:55:54.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:54 smithi132 ceph-mon[23807]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:54.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:54 smithi078 ceph-mon[22905]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:54.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:54 smithi078 ceph-mon[27629]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:56.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:56 smithi078 ceph-mon[22905]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:56.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:56 smithi078 ceph-mon[27629]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:56.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:56 smithi132 ceph-mon[23807]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:55:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:58 smithi078 ceph-mon[22905]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:55:58 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:58.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:58 smithi078 ceph-mon[27629]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:58.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:55:58 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:55:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:58 smithi132 ceph-mon[23807]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:55:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:55:58 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:00 smithi078 ceph-mon[22905]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:00.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:00 smithi078 ceph-mon[27629]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:00.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:00 smithi132 ceph-mon[23807]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:02.556 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:02 smithi132 ceph-mon[23807]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:02.676 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:02 smithi078 ceph-mon[22905]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:02.677 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:02 smithi078 ceph-mon[27629]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:03.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:02 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:56:02] "GET /metrics HTTP/1.1" 200 208020 "" "Prometheus/2.33.4" 2024-05-09T03:56:03.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:56:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:56:04.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:56:03.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:56:04.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:56:03.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:56:04.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:56:03.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:56:04.726 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:04 smithi132 ceph-mon[23807]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:04.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:04 smithi078 ceph-mon[22905]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:04.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:04 smithi078 ceph-mon[27629]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:06.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:06 smithi132 ceph-mon[23807]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:06.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:06 smithi078 ceph-mon[22905]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:06.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:06 smithi078 ceph-mon[27629]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:08 smithi132 ceph-mon[23807]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:08 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:08.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:08 smithi078 ceph-mon[22905]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:08.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:08 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:08.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:08 smithi078 ceph-mon[27629]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:08.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:08 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:10 smithi132 ceph-mon[23807]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:10.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:10 smithi078 ceph-mon[22905]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:10 smithi078 ceph-mon[27629]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:12.677 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:12 smithi078 ceph-mon[22905]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:12.677 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:12 smithi078 ceph-mon[27629]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:12.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:12 smithi132 ceph-mon[23807]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:13.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:56:12] "GET /metrics HTTP/1.1" 200 208020 "" "Prometheus/2.33.4" 2024-05-09T03:56:13.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:56:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:56:14.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:56:13.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:56:14.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:56:13.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:56:14.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:56:13.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:56:14.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:14 smithi132 ceph-mon[23807]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:14.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:14 smithi078 ceph-mon[22905]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:14.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:14 smithi078 ceph-mon[27629]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:56:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:15 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:56:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:56:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:15 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:56:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:56:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:15 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:56:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:56:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:15 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:56:15.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:56:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:15 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:56:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T03:56:15.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:15 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:56:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:16 smithi132 ceph-mon[23807]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:16 smithi078 ceph-mon[22905]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:16.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:16 smithi078 ceph-mon[27629]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:18.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:18 smithi132 ceph-mon[23807]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:18.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:18 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:18.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:18 smithi078 ceph-mon[22905]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:18.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:18 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:18.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:18 smithi078 ceph-mon[27629]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:18.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:18 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:20 smithi132 ceph-mon[23807]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:20.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:20 smithi078 ceph-mon[22905]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:20.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:20 smithi078 ceph-mon[27629]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:22.677 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:22 smithi078 ceph-mon[22905]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:22.677 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:22 smithi078 ceph-mon[27629]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:22.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:22 smithi132 ceph-mon[23807]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:23.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:22 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:56:22] "GET /metrics HTTP/1.1" 200 208020 "" "Prometheus/2.33.4" 2024-05-09T03:56:23.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:56:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:56:24.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:56:23.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:56:24.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:56:23.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:56:24.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:56:23.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:56:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:24 smithi132 ceph-mon[23807]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:24 smithi078 ceph-mon[22905]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:24 smithi078 ceph-mon[27629]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:26 smithi132 ceph-mon[23807]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:26 smithi078 ceph-mon[22905]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:26 smithi078 ceph-mon[27629]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:28 smithi132 ceph-mon[23807]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:28.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:28 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:28 smithi078 ceph-mon[22905]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:28.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:28 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:28 smithi078 ceph-mon[27629]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:28 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:30 smithi132 ceph-mon[23807]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:30.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:30 smithi078 ceph-mon[22905]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:30.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:30 smithi078 ceph-mon[27629]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:32.677 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:32 smithi078 ceph-mon[22905]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:32.677 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:32 smithi078 ceph-mon[27629]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:32.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:32 smithi132 ceph-mon[23807]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:33.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:32 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:56:32] "GET /metrics HTTP/1.1" 200 208020 "" "Prometheus/2.33.4" 2024-05-09T03:56:33.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:56:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:56:34.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:56:33.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:56:34.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:56:33.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:56:34.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:56:33.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:56:34.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:34 smithi132 ceph-mon[23807]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:34.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:34 smithi078 ceph-mon[22905]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:34.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:34 smithi078 ceph-mon[27629]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:36.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:36 smithi132 ceph-mon[23807]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:36.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:36 smithi078 ceph-mon[22905]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:36.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:36 smithi078 ceph-mon[27629]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:36.900 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T03:56:38.442 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:38 smithi078 ceph-mon[27629]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:38.442 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:38 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:38.442 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:38 smithi078 ceph-mon[22905]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:38.442 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:38 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:38.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:38 smithi132 ceph-mon[23807]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:38.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:38 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:38.858 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (6m) 2m ago 7m 25.4M - ba2b418f427c 1bc0d1c4e0c8 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (7m) 115s ago 7m 55.4M - 8.3.5 dad864ee21e9 c1b7ab086083 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (6m) 2m ago 6m 53.1M - 3.5 e1d6a67b021e 65c58273a7db 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283 running (2m) 115s ago 11m 67.5M - 19.0.0-3631-gc66c553a af4a4df59e0a 4a3f6038cf25 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:9283 running (13m) 2m ago 13m 459M - 17.2.0 e1d6a67b021e b2a9d11f22f2 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (13m) 2m ago 13m 50.0M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (12m) 115s ago 12m 42.9M 2048M 17.2.0 e1d6a67b021e a71bae52559d 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (12m) 2m ago 12m 43.9M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (8m) 2m ago 8m 18.9M - 1dbe0e931976 10d0785d9bf4 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (7m) 115s ago 7m 17.5M - 1dbe0e931976 dfbd85c54dbd 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (11m) 2m ago 11m 50.0M 4016M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (11m) 2m ago 11m 53.1M 4016M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T03:56:38.859 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (10m) 2m ago 10m 46.9M 4016M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T03:56:38.860 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (10m) 2m ago 10m 48.8M 4016M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T03:56:38.860 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (10m) 115s ago 9m 51.0M 4272M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T03:56:38.860 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (9m) 115s ago 9m 48.1M 4272M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T03:56:38.860 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (9m) 115s ago 9m 48.0M 4272M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T03:56:38.860 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (8m) 115s ago 8m 51.7M 4272M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T03:56:38.860 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (6m) 115s ago 8m 67.1M - 514e6a882f6e 69b608c5f2ac 2024-05-09T03:56:38.860 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (6m) 2m ago 6m 72.2M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T03:56:38.860 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (6m) 115s ago 6m 73.5M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T03:56:39.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions' 2024-05-09T03:56:40.639 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:40 smithi078 ceph-mon[27629]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:40.640 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:40 smithi078 ceph-mon[27629]: from='client.24910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:56:40.640 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:40 smithi078 ceph-mon[22905]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:40.640 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:40 smithi078 ceph-mon[22905]: from='client.24910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:56:40.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:40 smithi132 ceph-mon[23807]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:40.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:40 smithi132 ceph-mon[23807]: from='client.24910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:56:41.501 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T03:56:41.501 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T03:56:41.501 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-09T03:56:41.501 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:56:41.501 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T03:56:41.501 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-09T03:56:41.501 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 1 2024-05-09T03:56:41.501 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:56:41.501 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T03:56:41.501 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T03:56:41.502 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:56:41.502 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": {}, 2024-05-09T03:56:41.502 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T03:56:41.502 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T03:56:41.502 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T03:56:41.502 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T03:56:41.502 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-05-09T03:56:41.502 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 1 2024-05-09T03:56:41.502 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T03:56:41.502 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T03:56:42.160 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph -s' 2024-05-09T03:56:42.472 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:42 smithi078 ceph-mon[22905]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:42.472 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:42 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1665673144' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T03:56:42.473 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:42 smithi078 ceph-mon[27629]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:42.473 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:42 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1665673144' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T03:56:42.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:42 smithi132 ceph-mon[23807]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:42.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:42 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1665673144' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T03:56:42.762 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:42 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:56:42] "GET /metrics HTTP/1.1" 200 208020 "" "Prometheus/2.33.4" 2024-05-09T03:56:43.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:56:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:56:43.772 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:56:43.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:56:43.773 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:56:43.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:56:43.773 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:56:43.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:56:44.017 INFO:teuthology.orchestra.run.smithi078.stdout: cluster: 2024-05-09T03:56:44.017 INFO:teuthology.orchestra.run.smithi078.stdout: id: 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T03:56:44.017 INFO:teuthology.orchestra.run.smithi078.stdout: health: HEALTH_OK 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: services: 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: mgr: y(active, since 8m), standbys: x 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: data: 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: pools: 6 pools, 161 pgs 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: objects: 241 objects, 585 KiB 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: usage: 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: pgs: 161 active+clean 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: io: 2024-05-09T03:56:44.018 INFO:teuthology.orchestra.run.smithi078.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-09T03:56:44.019 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T03:56:44.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:44 smithi078 ceph-mon[27629]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:44.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:44 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:56:44.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:44 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:56:44.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:44 smithi078 ceph-mon[27629]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:56:44.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:44 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1545066482' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T03:56:44.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:44 smithi078 ceph-mon[27629]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:56:44.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:44 smithi078 ceph-mon[22905]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:44.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:44 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:56:44.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:44 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:56:44.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:44 smithi078 ceph-mon[22905]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:56:44.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:44 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1545066482' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T03:56:44.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:44 smithi078 ceph-mon[22905]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:56:44.606 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph health detail' 2024-05-09T03:56:44.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:44 smithi132 ceph-mon[23807]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:44.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:44 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:56:44.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:44 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:56:44.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:44 smithi132 ceph-mon[23807]: from='mgr.24340 172.21.15.78:0/3227295932' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:56:44.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:44 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1545066482' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T03:56:44.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:44 smithi132 ceph-mon[23807]: from='mgr.24340 ' entity='mgr.y' 2024-05-09T03:56:46.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:46 smithi078 ceph-mon[22905]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:46.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:46 smithi078 ceph-mon[27629]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:46.657 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-05-09T03:56:46.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:46 smithi132 ceph-mon[23807]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:47.272 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-05-09T03:56:47.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:47 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/271793409' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T03:56:47.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:47 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/271793409' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T03:56:47.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:47 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/271793409' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T03:56:48.617 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:48 smithi078 ceph-mon[22905]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:48.617 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:48 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:48.617 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:48 smithi078 ceph-mon[27629]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:48.617 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:48 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:48.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:48 smithi132 ceph-mon[23807]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:48.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:48 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:49.328 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T03:56:49.618 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:49 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/134687902' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T03:56:49.619 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:49 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/134687902' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T03:56:49.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:49 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/134687902' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T03:56:49.985 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph mgr fail' 2024-05-09T03:56:50.553 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:50 smithi078 ceph-mon[27629]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:50.553 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:50 smithi078 ceph-mon[22905]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:50.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:50 smithi132 ceph-mon[23807]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:56:52.662 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:52 smithi078 ceph-mon[22905]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:52.662 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:52 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2449170105' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T03:56:52.662 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:52 smithi078 ceph-mon[22905]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T03:56:52.663 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:52 smithi078 ceph-mon[22905]: osdmap e96: 8 total, 8 up, 8 in 2024-05-09T03:56:52.663 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:52.359+0000 7f806f234700 -1 mgr handle_mgr_map I was active but no longer am 2024-05-09T03:56:52.663 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ignoring --setuser ceph since I am not root 2024-05-09T03:56:52.663 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ignoring --setgroup ceph since I am not root 2024-05-09T03:56:52.663 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:52 smithi078 ceph-mgr[23144]: -- 172.21.15.78:0/967384137 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559b4a412340 con 0x559b4a3eac00 2024-05-09T03:56:52.663 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:52.528+0000 7fd4e0dd9000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-09T03:56:52.663 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:52 smithi078 ceph-mon[27629]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:52.663 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:52 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2449170105' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T03:56:52.664 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:52 smithi078 ceph-mon[27629]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T03:56:52.664 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:52 smithi078 ceph-mon[27629]: osdmap e96: 8 total, 8 up, 8 in 2024-05-09T03:56:52.693 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:56:52 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:56:52] ENGINE Bus STOPPING 2024-05-09T03:56:52.694 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:56:52 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:56:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-09T03:56:52.694 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:56:52 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:56:52] ENGINE Bus STOPPED 2024-05-09T03:56:52.694 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:56:52 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:56:52] ENGINE Bus STARTING 2024-05-09T03:56:52.694 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:56:52 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:56:52] ENGINE Serving on http://:::9283 2024-05-09T03:56:52.694 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:56:52 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:56:52] ENGINE Bus STARTED 2024-05-09T03:56:52.694 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:52 smithi132 ceph-mon[23807]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:56:52.694 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:52 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2449170105' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T03:56:52.694 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:52 smithi132 ceph-mon[23807]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T03:56:52.694 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:52 smithi132 ceph-mon[23807]: osdmap e96: 8 total, 8 up, 8 in 2024-05-09T03:56:52.993 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:52.663+0000 7fd4e0dd9000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:56:52.993 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:52 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:52.765+0000 7fd4e0dd9000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-09T03:56:53.019 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'sleep 180' 2024-05-09T03:56:53.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:53.093+0000 7fd4e0dd9000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-09T03:56:53.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:53.178+0000 7fd4e0dd9000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-09T03:56:53.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-09T03:56:53.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: mgrmap e22: x(active, starting, since 0.340059s) 2024-05-09T03:56:53.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:56:53.588 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:56:53.588 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:56:53.588 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T03:56:53.588 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:56:53.588 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:56:53.588 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:56:53.589 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:56:53.589 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:56:53.589 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:56:53.589 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:56:53.589 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:56:53.589 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T03:56:53.590 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T03:56:53.590 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T03:56:53.590 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: Manager daemon x is now available 2024-05-09T03:56:53.590 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:56:53.590 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: Queued rgw.foo for migration 2024-05-09T03:56:53.590 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-05-09T03:56:53.591 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:56:53.591 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:56:53.591 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:56:53.591 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:56:53.591 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:56:53.591 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-09T03:56:53.592 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-09T03:56:53.592 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-09T03:56:53.592 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: mgrmap e22: x(active, starting, since 0.340059s) 2024-05-09T03:56:53.592 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:56:53.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:56:53.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:56:53.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T03:56:53.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:56:53.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:56:53.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:56:53.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:56:53.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:56:53.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: Manager daemon x is now available 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: Queued rgw.foo for migration 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:56:53.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:56:53.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:56:53.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-09T03:56:53.595 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:53 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-09T03:56:53.595 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:56:53.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:56:53.595 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:56:53.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": dial tcp 172.21.15.78:8443: connect: connection refused" 2024-05-09T03:56:53.595 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:56:53 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:56:53.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:56:53.756 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:56:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:56:53] "GET /metrics HTTP/1.1" 200 34976 "" "Prometheus/2.33.4" 2024-05-09T03:56:53.757 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-09T03:56:53.757 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: mgrmap e22: x(active, starting, since 0.340059s) 2024-05-09T03:56:53.758 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T03:56:53.758 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T03:56:53.758 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T03:56:53.759 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T03:56:53.759 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T03:56:53.759 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T03:56:53.759 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T03:56:53.760 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T03:56:53.760 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T03:56:53.760 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T03:56:53.762 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T03:56:53.762 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T03:56:53.762 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T03:56:53.762 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T03:56:53.762 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T03:56:53.762 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: Manager daemon x is now available 2024-05-09T03:56:53.762 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:56:53.762 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: Queued rgw.foo for migration 2024-05-09T03:56:53.762 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-05-09T03:56:53.762 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:56:53.762 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:56:53.762 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:56:53.763 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:56:53.763 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-09T03:56:53.763 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-09T03:56:53.763 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:53 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-09T03:56:54.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:54 smithi078 ceph-mon[27629]: Deploying cephadm binary to smithi078 2024-05-09T03:56:54.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:54 smithi078 ceph-mon[27629]: Deploying cephadm binary to smithi132 2024-05-09T03:56:54.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:54 smithi078 ceph-mon[27629]: mgrmap e23: x(active, since 1.35482s) 2024-05-09T03:56:54.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:54 smithi078 ceph-mon[27629]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:54.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:54 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:56:54.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:54 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:56:54.595 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:54 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:54.331+0000 7fd4e0dd9000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-09T03:56:54.595 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:54 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:54.507+0000 7fd4e0dd9000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-09T03:56:54.595 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:54 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:54.592+0000 7fd4e0dd9000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-09T03:56:54.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:54 smithi078 ceph-mon[22905]: Deploying cephadm binary to smithi078 2024-05-09T03:56:54.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:54 smithi078 ceph-mon[22905]: Deploying cephadm binary to smithi132 2024-05-09T03:56:54.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:54 smithi078 ceph-mon[22905]: mgrmap e23: x(active, since 1.35482s) 2024-05-09T03:56:54.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:54 smithi078 ceph-mon[22905]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:54.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:54 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:56:54.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:54 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:56:54.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:54 smithi132 ceph-mon[23807]: Deploying cephadm binary to smithi078 2024-05-09T03:56:54.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:54 smithi132 ceph-mon[23807]: Deploying cephadm binary to smithi132 2024-05-09T03:56:54.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:54 smithi132 ceph-mon[23807]: mgrmap e23: x(active, since 1.35482s) 2024-05-09T03:56:54.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:54 smithi132 ceph-mon[23807]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:54.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:54 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:56:54.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:54 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:56:55.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:54 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:54.920+0000 7fd4e0dd9000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-09T03:56:55.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:55 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:55.012+0000 7fd4e0dd9000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-09T03:56:55.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:55 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:55.105+0000 7fd4e0dd9000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-09T03:56:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:55 smithi132 ceph-mon[23807]: [09/May/2024:03:56:54] ENGINE Bus STARTING 2024-05-09T03:56:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:55 smithi132 ceph-mon[23807]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:55 smithi132 ceph-mon[23807]: [09/May/2024:03:56:54] ENGINE Serving on https://172.21.15.132:7150 2024-05-09T03:56:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:55 smithi132 ceph-mon[23807]: [09/May/2024:03:56:54] ENGINE Serving on http://172.21.15.132:8765 2024-05-09T03:56:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:55 smithi132 ceph-mon[23807]: [09/May/2024:03:56:54] ENGINE Bus STARTED 2024-05-09T03:56:55.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:55 smithi132 ceph-mon[23807]: mgrmap e24: x(active, since 2s) 2024-05-09T03:56:55.762 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:55 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:55.349+0000 7fd4e0dd9000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-09T03:56:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:55 smithi078 ceph-mon[22905]: [09/May/2024:03:56:54] ENGINE Bus STARTING 2024-05-09T03:56:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:55 smithi078 ceph-mon[22905]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:55 smithi078 ceph-mon[22905]: [09/May/2024:03:56:54] ENGINE Serving on https://172.21.15.132:7150 2024-05-09T03:56:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:55 smithi078 ceph-mon[22905]: [09/May/2024:03:56:54] ENGINE Serving on http://172.21.15.132:8765 2024-05-09T03:56:55.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:55 smithi078 ceph-mon[22905]: [09/May/2024:03:56:54] ENGINE Bus STARTED 2024-05-09T03:56:55.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:55 smithi078 ceph-mon[22905]: mgrmap e24: x(active, since 2s) 2024-05-09T03:56:55.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:55 smithi078 ceph-mon[27629]: [09/May/2024:03:56:54] ENGINE Bus STARTING 2024-05-09T03:56:55.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:55 smithi078 ceph-mon[27629]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:55.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:55 smithi078 ceph-mon[27629]: [09/May/2024:03:56:54] ENGINE Serving on https://172.21.15.132:7150 2024-05-09T03:56:55.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:55 smithi078 ceph-mon[27629]: [09/May/2024:03:56:54] ENGINE Serving on http://172.21.15.132:8765 2024-05-09T03:56:55.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:55 smithi078 ceph-mon[27629]: [09/May/2024:03:56:54] ENGINE Bus STARTED 2024-05-09T03:56:55.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:55 smithi078 ceph-mon[27629]: mgrmap e24: x(active, since 2s) 2024-05-09T03:56:56.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:55 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:55.790+0000 7fd4e0dd9000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-09T03:56:56.671 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:56 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:56.278+0000 7fd4e0dd9000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-09T03:56:56.671 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:56 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:56.373+0000 7fd4e0dd9000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-09T03:56:57.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:56 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:56.669+0000 7fd4e0dd9000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-09T03:56:57.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:56 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:56.760+0000 7fd4e0dd9000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-09T03:56:57.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:57 smithi078 ceph-mon[27629]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:57.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:57 smithi078 ceph-mon[27629]: mgrmap e25: x(active, since 4s) 2024-05-09T03:56:57.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:57 smithi078 ceph-mon[22905]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:57.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:57 smithi078 ceph-mon[22905]: mgrmap e25: x(active, since 4s) 2024-05-09T03:56:57.513 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:57 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:57.168+0000 7fd4e0dd9000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-09T03:56:57.513 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:57 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:57.302+0000 7fd4e0dd9000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-09T03:56:57.513 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:57 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:57.397+0000 7fd4e0dd9000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-09T03:56:57.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:57 smithi132 ceph-mon[23807]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:57.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:57 smithi132 ceph-mon[23807]: mgrmap e25: x(active, since 4s) 2024-05-09T03:56:58.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:57 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:57.515+0000 7fd4e0dd9000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-09T03:56:58.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:57 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:57.605+0000 7fd4e0dd9000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-09T03:56:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:58 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:58.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:58.196+0000 7fd4e0dd9000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-09T03:56:58.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:58 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:58 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:56:59.016 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:58.572+0000 7fd4e0dd9000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-09T03:56:59.016 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:56:58] ENGINE Bus STARTING 2024-05-09T03:56:59.016 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: CherryPy Checker: 2024-05-09T03:56:59.016 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: The Application mounted at '' has an empty config. 2024-05-09T03:56:59.016 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: 2024-05-09T03:56:59.016 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:56:58] ENGINE Serving on http://:::9283 2024-05-09T03:56:59.016 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:56:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: [09/May/2024:03:56:58] ENGINE Bus STARTED 2024-05-09T03:56:59.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:59 smithi132 ceph-mon[23807]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:59.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:59 smithi132 ceph-mon[23807]: Standby manager daemon y started 2024-05-09T03:56:59.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:59 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.78:0/2175483657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-09T03:56:59.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:59 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.78:0/2175483657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T03:56:59.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:59 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.78:0/2175483657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-09T03:56:59.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:56:59 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.78:0/2175483657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T03:56:59.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:59 smithi078 ceph-mon[22905]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:59.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:59 smithi078 ceph-mon[22905]: Standby manager daemon y started 2024-05-09T03:56:59.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:59 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.78:0/2175483657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-09T03:56:59.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:59 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.78:0/2175483657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T03:56:59.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:59 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.78:0/2175483657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-09T03:56:59.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:56:59 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.78:0/2175483657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T03:56:59.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:59 smithi078 ceph-mon[27629]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:56:59.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:59 smithi078 ceph-mon[27629]: Standby manager daemon y started 2024-05-09T03:56:59.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:59 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.78:0/2175483657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-09T03:56:59.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:59 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.78:0/2175483657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T03:56:59.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:59 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.78:0/2175483657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-09T03:56:59.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:56:59 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.78:0/2175483657' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T03:57:00.404 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:00 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:57:00.225Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:57:00.404 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:00 smithi132 ceph-mon[23807]: mgrmap e26: x(active, since 7s), standbys: y 2024-05-09T03:57:00.405 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:00 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T03:57:00.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:00 smithi078 ceph-mon[22905]: mgrmap e26: x(active, since 7s), standbys: y 2024-05-09T03:57:00.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:00 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T03:57:00.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:00 smithi078 ceph-mon[27629]: mgrmap e26: x(active, since 7s), standbys: y 2024-05-09T03:57:00.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:00 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T03:57:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:01 smithi132 ceph-mon[23807]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T03:57:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:01 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:01.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:01 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:01 smithi078 ceph-mon[22905]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T03:57:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:01 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:01 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:01.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:01 smithi078 ceph-mon[27629]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T03:57:01.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:01 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:01.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:01 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:57:02 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:57:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:57:03.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: Adjusting osd_memory_target on smithi078 to 2992M 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[27629]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T03:57:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: Adjusting osd_memory_target on smithi078 to 2992M 2024-05-09T03:57:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:02 smithi078 ceph-mon[22905]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T03:57:03.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: Adjusting osd_memory_target on smithi078 to 2992M 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:03.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:02 smithi132 ceph-mon[23807]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T03:57:03.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:57:03] "GET /metrics HTTP/1.1" 200 34976 "" "Prometheus/2.33.4" 2024-05-09T03:57:03.981 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=error ts=2024-05-09T03:57:03.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:57:03.981 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:57:03.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.78:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.78 because it doesn't contain any IP SANs" 2024-05-09T03:57:03.981 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=warn ts=2024-05-09T03:57:03.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.132:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.132 because it doesn't contain any IP SANs" 2024-05-09T03:57:04.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:04.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:04.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: Adjusting osd_memory_target on smithi132 to 3248M 2024-05-09T03:57:04.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:04.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:57:04.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:57:04.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T03:57:04.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T03:57:04.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T03:57:04.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T03:57:04.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:57:04.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:03 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:57:04.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:04.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:04.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: Adjusting osd_memory_target on smithi132 to 3248M 2024-05-09T03:57:04.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:04.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:57:04.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:57:04.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T03:57:04.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T03:57:04.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T03:57:04.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T03:57:04.266 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:57:04.266 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:57:04.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:04.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:04.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-09T03:57:04.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: Adjusting osd_memory_target on smithi132 to 3248M 2024-05-09T03:57:04.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:04.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:57:04.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:57:04.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T03:57:04.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T03:57:04.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T03:57:04.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T03:57:04.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:57:04.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:03 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T03:57:05.012 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:04 smithi078 systemd[1]: Stopping Ceph node-exporter.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:57:05.012 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:04 smithi078 bash[56590]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter.a" found: no such container 2024-05-09T03:57:05.013 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:04 smithi078 podman[56603]: 2024-05-09 03:57:04.851387316 +0000 UTC m=+0.127084725 container died 10d0785d9bf451416a9ec8973c6fa5291265276fc02dfc788d28eb5e32432bde (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:05.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T03:57:05.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T03:57:05.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: Reconfiguring node-exporter.a (dependencies changed)... 2024-05-09T03:57:05.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: Reconfiguring daemon node-exporter.a on smithi078 2024-05-09T03:57:05.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:05 smithi132 ceph-mon[23807]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T03:57:05.512 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 podman[56603]: 2024-05-09 03:57:05.198870616 +0000 UTC m=+0.474568015 container cleanup 10d0785d9bf451416a9ec8973c6fa5291265276fc02dfc788d28eb5e32432bde (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:05.513 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 bash[56603]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a 2024-05-09T03:57:05.513 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 bash[56628]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter.a" found: no such container 2024-05-09T03:57:05.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T03:57:05.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T03:57:05.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: Reconfiguring node-exporter.a (dependencies changed)... 2024-05-09T03:57:05.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: Reconfiguring daemon node-exporter.a on smithi078 2024-05-09T03:57:05.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[27629]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T03:57:05.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T03:57:05.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T03:57:05.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.515 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.515 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.515 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.515 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.515 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.515 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.515 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:05.515 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: Reconfiguring node-exporter.a (dependencies changed)... 2024-05-09T03:57:05.515 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: Reconfiguring daemon node-exporter.a on smithi078 2024-05-09T03:57:05.515 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-mon[22905]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T03:57:05.875 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 podman[56617]: 2024-05-09 03:57:05.53644903 +0000 UTC m=+0.684861048 container remove 10d0785d9bf451416a9ec8973c6fa5291265276fc02dfc788d28eb5e32432bde (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:05.875 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-05-09T03:57:05.875 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-05-09T03:57:05.875 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 systemd[1]: Stopped Ceph node-exporter.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:57:05.875 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.a.service: Consumed 3.661s CPU time. 2024-05-09T03:57:05.875 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 systemd[1]: Starting Ceph node-exporter.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:57:05.875 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 podman[56700]: 2024-05-09 03:57:05.669750779 +0000 UTC m=+0.017785905 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-09T03:57:05.876 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 podman[56700]: 2024-05-09 03:57:05.7680107 +0000 UTC m=+0.116045806 container create 4a3cc1414851bdd6a1171351bed2e3d0d00f54d6c0b55571568cf9dc60f02a7f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:06.262 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 podman[56700]: 2024-05-09 03:57:05.953758561 +0000 UTC m=+0.301793673 container init 4a3cc1414851bdd6a1171351bed2e3d0d00f54d6c0b55571568cf9dc60f02a7f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 podman[56700]: 2024-05-09 03:57:05.956711227 +0000 UTC m=+0.304746335 container start 4a3cc1414851bdd6a1171351bed2e3d0d00f54d6c0b55571568cf9dc60f02a7f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.960Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.960Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.960Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.960Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=arp 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-09T03:57:06.263 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=edac 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-09T03:57:06.264 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-09T03:57:06.265 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-09T03:57:06.265 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-09T03:57:06.265 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-09T03:57:06.265 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-09T03:57:06.265 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=os 2024-05-09T03:57:06.265 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-09T03:57:06.265 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-09T03:57:06.265 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-09T03:57:06.265 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-09T03:57:06.265 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-09T03:57:06.265 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=stat 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=time 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=uname 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[56715]: ts=2024-05-09T03:57:05.961Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:06 smithi078 bash[56700]: 4a3cc1414851bdd6a1171351bed2e3d0d00f54d6c0b55571568cf9dc60f02a7f 2024-05-09T03:57:06.266 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 03:57:06 smithi078 systemd[1]: Started Ceph node-exporter.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:57:07.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:07 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:07.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:07 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:07.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:07 smithi078 ceph-mon[22905]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-09T03:57:07.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:07 smithi078 ceph-mon[22905]: Reconfiguring daemon alertmanager.a on smithi078 2024-05-09T03:57:07.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:07 smithi078 ceph-mon[22905]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T03:57:07.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:07 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:07.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:07 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:07.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:07 smithi078 ceph-mon[27629]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-09T03:57:07.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:07 smithi078 ceph-mon[27629]: Reconfiguring daemon alertmanager.a on smithi078 2024-05-09T03:57:07.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:07 smithi078 ceph-mon[27629]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T03:57:07.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:07 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:07.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:07 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:07.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:07 smithi132 ceph-mon[23807]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-09T03:57:07.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:07 smithi132 ceph-mon[23807]: Reconfiguring daemon alertmanager.a on smithi078 2024-05-09T03:57:07.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:07 smithi132 ceph-mon[23807]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T03:57:08.325 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:08 smithi078 ceph-mon[22905]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:57:08.325 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:08 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:57:08.326 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:08 smithi078 ceph-mon[27629]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:57:08.326 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:08 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:57:08.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:08 smithi132 ceph-mon[23807]: from='client.24611 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:57:08.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:08 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:57:09.359 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:09 smithi078 ceph-mon[22905]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T03:57:09.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:09 smithi078 ceph-mon[27629]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T03:57:09.360 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:09 smithi078 systemd[1]: Stopping Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:57:09.360 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:09 smithi078 bash[56888]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager.a" found: no such container 2024-05-09T03:57:09.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:09 smithi132 ceph-mon[23807]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T03:57:09.694 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[47436]: level=info ts=2024-05-09T03:57:09.400Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-05-09T03:57:09.694 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:09 smithi078 podman[56901]: 2024-05-09 03:57:09.420641746 +0000 UTC m=+0.131276505 container died 1bc0d1c4e0c8c5b06bdc493a68a7a354fc3b669398622db4a4cce63f62b2a47a (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:09.968 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:09 smithi078 podman[56901]: 2024-05-09 03:57:09.760300336 +0000 UTC m=+0.470935099 container cleanup 1bc0d1c4e0c8c5b06bdc493a68a7a354fc3b669398622db4a4cce63f62b2a47a (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:09.968 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:09 smithi078 bash[56901]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a 2024-05-09T03:57:10.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:09 smithi078 podman[56916]: 2024-05-09 03:57:09.968093708 +0000 UTC m=+0.542227148 container remove 1bc0d1c4e0c8c5b06bdc493a68a7a354fc3b669398622db4a4cce63f62b2a47a (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:10.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 podman[56916]: 2024-05-09 03:57:10.077148643 +0000 UTC m=+0.651282091 volume remove 61d1f91c2fc39cda2db7a61508c6c2728a6e5b514eca5977fadfeccc7079faeb 2024-05-09T03:57:10.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 bash[56936]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager.a" found: no such container 2024-05-09T03:57:10.262 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-09T03:57:10.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 systemd[1]: Stopped Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:57:10.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@alertmanager.a.service: Consumed 1.036s CPU time. 2024-05-09T03:57:10.263 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 systemd[1]: Starting Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:57:10.639 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 podman[57008]: 2024-05-09 03:57:10.26296165 +0000 UTC m=+0.018276162 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-05-09T03:57:10.639 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 podman[57008]: 2024-05-09 03:57:10.369788066 +0000 UTC m=+0.125102572 volume create a1684abc154aa6fab9d4672b25496da89fddb483be351c009e6b16ae17dc2af8 2024-05-09T03:57:10.639 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 podman[57008]: 2024-05-09 03:57:10.453551025 +0000 UTC m=+0.208865524 container create f44cb50f5f358f021ca163840b8438a1783603d2554dc0078f0a6647a78c25c6 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:11.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 podman[57008]: 2024-05-09 03:57:10.756448293 +0000 UTC m=+0.511762791 container init f44cb50f5f358f021ca163840b8438a1783603d2554dc0078f0a6647a78c25c6 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:11.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 podman[57008]: 2024-05-09 03:57:10.759499228 +0000 UTC m=+0.514813728 container start f44cb50f5f358f021ca163840b8438a1783603d2554dc0078f0a6647a78c25c6 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:11.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[57024]: level=info ts=2024-05-09T03:57:10.769Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-05-09T03:57:11.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[57024]: level=info ts=2024-05-09T03:57:10.769Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-05-09T03:57:11.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[57024]: level=info ts=2024-05-09T03:57:10.771Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.78 port=9094 2024-05-09T03:57:11.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[57024]: level=info ts=2024-05-09T03:57:10.771Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-09T03:57:11.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[57024]: level=info ts=2024-05-09T03:57:10.795Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-09T03:57:11.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[57024]: level=info ts=2024-05-09T03:57:10.796Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-09T03:57:11.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[57024]: level=info ts=2024-05-09T03:57:10.798Z caller=main.go:518 msg=Listening address=:9093 2024-05-09T03:57:11.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[57024]: level=info ts=2024-05-09T03:57:10.798Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-05-09T03:57:11.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 bash[57008]: f44cb50f5f358f021ca163840b8438a1783603d2554dc0078f0a6647a78c25c6 2024-05-09T03:57:11.013 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:10 smithi078 systemd[1]: Started Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:57:11.719 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:11 smithi078 ceph-mon[22905]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T03:57:11.719 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:11 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:11.719 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:11 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:11.719 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:11 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:57:11.719 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:11 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:57:11.719 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:11 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:57:11.719 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:11 smithi078 ceph-mon[27629]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T03:57:11.719 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:11 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:11.719 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:11 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:11.719 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:11 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:57:11.719 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:11 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:57:11.720 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:11 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:57:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:11 smithi132 ceph-mon[23807]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T03:57:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:11 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:11 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:11 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:57:11.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:11 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:57:11.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:11 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:57:12.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:12 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.78:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.132:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:57:12.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:12 smithi132 ceph-mon[23807]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T03:57:12.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:12 smithi132 ceph-mon[23807]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:57:12.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:12 smithi078 ceph-mon[27629]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T03:57:12.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:12 smithi078 ceph-mon[27629]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:57:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:12 smithi078 ceph-mon[22905]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T03:57:12.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:12 smithi078 ceph-mon[22905]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:57:12.763 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 03:57:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[23140]: ::ffff:172.21.15.132 - - [09/May/2024:03:57:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T03:57:13.058 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[57024]: level=info ts=2024-05-09T03:57:12.772Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000171332s 2024-05-09T03:57:13.723 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:13 smithi078 ceph-mon[22905]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:57:13.723 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:13 smithi078 ceph-mon[27629]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:57:13.754 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:57:13] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-05-09T03:57:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:13 smithi132 ceph-mon[23807]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:57:14.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T03:57:14.140+0000 7f4ffcefb700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Creating ceph-iscsi config... 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return _run_code(code, main_globals, None, 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: exec(code, run_globals) 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:14.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:14.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:14.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:14.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:14.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:14.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:14.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:14.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T03:57:14.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return future.result(timeout) 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return self.__get_result() 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: raise self._exception 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: use_current_daemon_image=reconfig, 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:57:14.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Creating ceph-iscsi config... 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return _run_code(code, main_globals, None, 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: exec(code, run_globals) 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:14.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:14.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:14.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:14.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:14.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:14.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:14.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:14.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:14.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T03:57:14.142+0000 7f4ffcefb700 -1 log_channel(cephadm) log [ERR] : Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T03:57:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:15 smithi132 ceph-mon[23807]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout: Creating ceph-iscsi config... 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: exec(code, run_globals) 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: return future.result(timeout) 2024-05-09T03:57:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: return self.__get_result() 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: raise self._exception 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: use_current_daemon_image=reconfig, 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Creating ceph-iscsi config... 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: exec(code, run_globals) 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:15 smithi132 ceph-mon[23807]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T03:57:15.759 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:15 smithi132 ceph-mon[23807]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T03:57:15.759 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:15 smithi132 ceph-mon[23807]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T03:57:15.759 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:15 smithi132 ceph-mon[23807]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-09T03:57:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:15 smithi078 ceph-mon[22905]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:57:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: return future.result(timeout) 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:57:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: return self.__get_result() 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: raise self._exception 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: use_current_daemon_image=reconfig, 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:15 smithi078 ceph-mon[22905]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T03:57:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:15 smithi078 ceph-mon[22905]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T03:57:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:15 smithi078 ceph-mon[22905]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T03:57:15.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:15 smithi078 ceph-mon[22905]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-09T03:57:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:15 smithi078 ceph-mon[27629]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:57:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: return future.result(timeout) 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: return self.__get_result() 2024-05-09T03:57:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: raise self._exception 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: use_current_daemon_image=reconfig, 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:57:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:15 smithi078 ceph-mon[27629]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:15 smithi078 ceph-mon[27629]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T03:57:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:15 smithi078 ceph-mon[27629]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T03:57:15.772 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:15 smithi078 ceph-mon[27629]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-09T03:57:17.723 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:17 smithi132 ceph-mon[23807]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:57:17.723 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 systemd[1]: Stopping Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:57:17.723 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 bash[47073]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus.a" found: no such container 2024-05-09T03:57:17.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:17 smithi078 ceph-mon[22905]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:57:17.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:17 smithi078 ceph-mon[27629]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:57:18.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:17.721Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-09T03:57:18.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:17.721Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-09T03:57:18.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:17.721Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-09T03:57:18.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:17.721Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-09T03:57:18.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:17.721Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-09T03:57:18.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:17.721Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-09T03:57:18.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:17.721Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-09T03:57:18.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:17.721Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-09T03:57:18.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:17.721Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-09T03:57:18.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:17.722Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-09T03:57:18.007 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:17.722Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-09T03:57:18.007 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[40706]: ts=2024-05-09T03:57:17.722Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-09T03:57:18.007 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:17 smithi132 podman[47086]: 2024-05-09 03:57:17.731472736 +0000 UTC m=+0.113420509 container died 69b608c5f2ac802d5bc4fa19119d4e3ba140b2f223b30768ee1ca9c12d8cb605 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:18.389 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 podman[47086]: 2024-05-09 03:57:18.112313473 +0000 UTC m=+0.494261245 container cleanup 69b608c5f2ac802d5bc4fa19119d4e3ba140b2f223b30768ee1ca9c12d8cb605 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:18.389 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 bash[47086]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a 2024-05-09T03:57:18.389 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 bash[47114]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus.a" found: no such container 2024-05-09T03:57:18.656 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 podman[47101]: 2024-05-09 03:57:18.388321658 +0000 UTC m=+0.658591252 container remove 69b608c5f2ac802d5bc4fa19119d4e3ba140b2f223b30768ee1ca9c12d8cb605 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:18.656 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-09T03:57:18.657 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 systemd[1]: Stopped Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:57:18.657 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service: Consumed 1.932s CPU time. 2024-05-09T03:57:18.657 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 systemd[1]: Starting Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:57:18.657 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 podman[47185]: 2024-05-09 03:57:18.52538081 +0000 UTC m=+0.019259053 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-09T03:57:18.934 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 podman[47185]: 2024-05-09 03:57:18.656225155 +0000 UTC m=+0.150103393 container create 860656079d4cfb4fe4ee4fbe28d82fe5500220e84181752e5d2e876b60acefdc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:19.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:19 smithi132 ceph-mon[23807]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:57:19.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 podman[47185]: 2024-05-09 03:57:18.933545235 +0000 UTC m=+0.427423473 container init 860656079d4cfb4fe4ee4fbe28d82fe5500220e84181752e5d2e876b60acefdc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:19.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 podman[47185]: 2024-05-09 03:57:18.93648905 +0000 UTC m=+0.430367293 container start 860656079d4cfb4fe4ee4fbe28d82fe5500220e84181752e5d2e876b60acefdc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T03:57:19.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:18.960Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-09T03:57:19.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:18.960Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-09T03:57:19.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:18.960Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-09T03:57:19.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:18.960Z caller=main.go:518 level=info host_details="(Linux 5.14.0-444.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024 x86_64 smithi132 (none))" 2024-05-09T03:57:19.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:18.960Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-09T03:57:19.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:18.960Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-09T03:57:19.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:18.962Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-09T03:57:19.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:18.962Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-09T03:57:19.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:18.963Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-09T03:57:19.257 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:18.965Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-09T03:57:19.257 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:18.965Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.542µs 2024-05-09T03:57:19.257 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:18.965Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-09T03:57:19.257 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:19 smithi132 bash[47185]: 860656079d4cfb4fe4ee4fbe28d82fe5500220e84181752e5d2e876b60acefdc 2024-05-09T03:57:19.257 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:19 smithi132 systemd[1]: Started Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:57:19.257 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:19.168Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-05-09T03:57:19.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:19 smithi078 ceph-mon[22905]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:57:19.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:19 smithi078 ceph-mon[27629]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:57:19.632 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:19.438Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-05-09T03:57:19.632 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:19.438Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-05-09T03:57:19.632 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:19.438Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.278µs wal_replay_duration=472.78444ms total_replay_duration=472.826584ms 2024-05-09T03:57:19.632 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:19.441Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-09T03:57:19.632 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:19.441Z caller=main.go:947 level=info msg="TSDB started" 2024-05-09T03:57:19.632 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:19.441Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-09T03:57:19.632 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:19.467Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.934538ms db_storage=781ns remote_storage=1.161µs web_handler=299ns query_engine=758ns scrape=374.841µs scrape_sd=94.011µs notify=16.624µs notify_sd=16.07µs rules=25.036222ms 2024-05-09T03:57:19.633 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:19.467Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-09T03:57:20.005 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:19 smithi132 systemd[1]: Stopping Ceph node-exporter.b for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:57:20.005 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:19 smithi132 bash[47319]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter.b" found: no such container 2024-05-09T03:57:20.006 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:19 smithi132 podman[47332]: 2024-05-09 03:57:19.797982272 +0000 UTC m=+0.132363206 container died dfbd85c54dbd62548e5201ec54d0f9b84cb0a3110587dce8a86b0ed460250c59 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T03:57:20.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:20 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:20.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:20 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:20.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:20 smithi078 ceph-mon[22905]: Reconfiguring node-exporter.b (dependencies changed)... 2024-05-09T03:57:20.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:20 smithi078 ceph-mon[22905]: Reconfiguring daemon node-exporter.b on smithi132 2024-05-09T03:57:20.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:20 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:20.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:20 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:20.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:20 smithi078 ceph-mon[27629]: Reconfiguring node-exporter.b (dependencies changed)... 2024-05-09T03:57:20.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:20 smithi078 ceph-mon[27629]: Reconfiguring daemon node-exporter.b on smithi132 2024-05-09T03:57:20.456 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 podman[47332]: 2024-05-09 03:57:20.153978235 +0000 UTC m=+0.488359185 container cleanup dfbd85c54dbd62548e5201ec54d0f9b84cb0a3110587dce8a86b0ed460250c59 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T03:57:20.456 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 bash[47332]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b 2024-05-09T03:57:20.456 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 bash[47360]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter.b" found: no such container 2024-05-09T03:57:20.456 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:20.456 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:20.457 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-mon[23807]: Reconfiguring node-exporter.b (dependencies changed)... 2024-05-09T03:57:20.457 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-mon[23807]: Reconfiguring daemon node-exporter.b on smithi132 2024-05-09T03:57:20.754 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 podman[47348]: 2024-05-09 03:57:20.455490509 +0000 UTC m=+0.661489817 container remove dfbd85c54dbd62548e5201ec54d0f9b84cb0a3110587dce8a86b0ed460250c59 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T03:57:20.754 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-05-09T03:57:20.755 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-05-09T03:57:20.755 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 systemd[1]: Stopped Ceph node-exporter.b for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:57:20.755 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.b.service: Consumed 3.764s CPU time. 2024-05-09T03:57:20.755 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 systemd[1]: Starting Ceph node-exporter.b for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:57:20.755 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 podman[47429]: 2024-05-09 03:57:20.5896481 +0000 UTC m=+0.017866769 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-09T03:57:20.755 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 podman[47429]: 2024-05-09 03:57:20.698141173 +0000 UTC m=+0.126359842 container create 666630a5311453012e0ff03c4836df7fc38870fcde3d09d8822e79ffefc6608c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T03:57:21.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 03:57:20 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[57024]: level=info ts=2024-05-09T03:57:20.774Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002471686s 2024-05-09T03:57:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:21 smithi132 ceph-mon[23807]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:57:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:21 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:21.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:21 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:21.255 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 podman[47429]: 2024-05-09 03:57:20.941860137 +0000 UTC m=+0.370078806 container init 666630a5311453012e0ff03c4836df7fc38870fcde3d09d8822e79ffefc6608c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T03:57:21.255 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 podman[47429]: 2024-05-09 03:57:20.944585446 +0000 UTC m=+0.372804118 container start 666630a5311453012e0ff03c4836df7fc38870fcde3d09d8822e79ffefc6608c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T03:57:21.255 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.948Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-09T03:57:21.255 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.948Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-09T03:57:21.255 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-09T03:57:21.255 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-09T03:57:21.255 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=arp 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=edac 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-09T03:57:21.256 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.949Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=os 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-09T03:57:21.257 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=stat 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=time 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=uname 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-09T03:57:21.258 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-09T03:57:21.259 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-09T03:57:21.259 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[47443]: ts=2024-05-09T03:57:20.950Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-09T03:57:21.259 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:21 smithi132 bash[47429]: 666630a5311453012e0ff03c4836df7fc38870fcde3d09d8822e79ffefc6608c 2024-05-09T03:57:21.259 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 03:57:21 smithi132 systemd[1]: Started Ceph node-exporter.b for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:57:21.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:21 smithi078 ceph-mon[27629]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:57:21.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:21 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:21.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:21 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:21.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:21 smithi078 ceph-mon[22905]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:57:21.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:21 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:21.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:21 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:22.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:22 smithi132 ceph-mon[23807]: Reconfiguring grafana.a (dependencies changed)... 2024-05-09T03:57:22.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:22 smithi132 ceph-mon[23807]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-09T03:57:22.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:22 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:22.466 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:22 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:22.467 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:22 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-09T03:57:22.467 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:22 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-09T03:57:22.467 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:22 smithi132 ceph-mon[23807]: Reconfiguring daemon grafana.a on smithi132 2024-05-09T03:57:22.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[22905]: Reconfiguring grafana.a (dependencies changed)... 2024-05-09T03:57:22.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[22905]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-09T03:57:22.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:22.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:22.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-09T03:57:22.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-09T03:57:22.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[22905]: Reconfiguring daemon grafana.a on smithi132 2024-05-09T03:57:22.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[27629]: Reconfiguring grafana.a (dependencies changed)... 2024-05-09T03:57:22.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[27629]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-09T03:57:22.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:22.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:22.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-09T03:57:22.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-09T03:57:22.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:22 smithi078 ceph-mon[27629]: Reconfiguring daemon grafana.a on smithi132 2024-05-09T03:57:23.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:23 smithi132 ceph-mon[23807]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:57:23.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:23 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:57:23.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:23 smithi078 ceph-mon[22905]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:57:23.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:23 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:57:23.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:23 smithi078 ceph-mon[27629]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:57:23.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:23 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:57:24.229 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:24 smithi132 systemd[1]: Stopping Ceph grafana.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:57:24.504 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:24 smithi132 bash[47615]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana.a" found: no such container 2024-05-09T03:57:24.505 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[39918]: t=2024-05-09T03:57:24+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-05-09T03:57:24.505 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:24 smithi132 podman[47627]: 2024-05-09 03:57:24.370968695 +0000 UTC m=+0.132729416 container died c1b7ab0860834ba92f21f2cfaa0faf2959435ebe2d6cfaf35a5df9115dd36c2c (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, distribution-scope=public, name=ubi8, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-type=git, vendor=Red Hat, Inc., io.buildah.version=1.24.2, version=8.5, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, description=Ceph Grafana Container) 2024-05-09T03:57:24.913 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:24 smithi132 podman[47627]: 2024-05-09 03:57:24.760146757 +0000 UTC m=+0.521907476 container cleanup c1b7ab0860834ba92f21f2cfaa0faf2959435ebe2d6cfaf35a5df9115dd36c2c (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, architecture=x86_64, description=Ceph Grafana Container, distribution-scope=public, io.openshift.tags=base rhel8, release=236.1648460182, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , version=8.5, io.openshift.expose-services=, name=ubi8, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-09T03:57:24.913 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:24 smithi132 bash[47627]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a 2024-05-09T03:57:24.914 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:24 smithi132 bash[47654]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana.a" found: no such container 2024-05-09T03:57:25.238 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:57:24] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-05-09T03:57:25.317 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 podman[47641]: 2024-05-09 03:57:25.002901424 +0000 UTC m=+0.625321746 container remove c1b7ab0860834ba92f21f2cfaa0faf2959435ebe2d6cfaf35a5df9115dd36c2c (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, build-date=2022-03-28T10:36:18.413762, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., description=Ceph Grafana Container, io.openshift.expose-services=, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, io.buildah.version=1.24.2, release=236.1648460182, architecture=x86_64) 2024-05-09T03:57:25.317 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@grafana.a.service: Deactivated successfully. 2024-05-09T03:57:25.317 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 systemd[1]: Stopped Ceph grafana.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:57:25.317 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@grafana.a.service: Consumed 3.283s CPU time. 2024-05-09T03:57:25.317 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 systemd[1]: Starting Ceph grafana.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T03:57:25.505 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 podman[47725]: 2024-05-09 03:57:25.139329903 +0000 UTC m=+0.017289029 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-05-09T03:57:25.505 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 podman[47725]: 2024-05-09 03:57:25.245411733 +0000 UTC m=+0.123370878 container create 33b2a450dbe16d5881623fba7157c2dc56aa81948aa49aaabdbffd738a1fa12e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, distribution-scope=public, io.openshift.tags=base rhel8, name=ubi8, build-date=2022-03-28T10:36:18.413762, release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, architecture=x86_64, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vendor=Red Hat, Inc.) 2024-05-09T03:57:25.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:25 smithi132 ceph-mon[23807]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:57:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:25 smithi078 ceph-mon[22905]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:57:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:25 smithi078 ceph-mon[27629]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:57:25.834 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Bus STOPPING 2024-05-09T03:57:25.834 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-09T03:57:25.835 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Bus STOPPED 2024-05-09T03:57:25.835 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Bus STARTING 2024-05-09T03:57:25.835 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Serving on http://:::9283 2024-05-09T03:57:25.835 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 podman[47725]: 2024-05-09 03:57:25.526311681 +0000 UTC m=+0.404270781 container init 33b2a450dbe16d5881623fba7157c2dc56aa81948aa49aaabdbffd738a1fa12e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, distribution-scope=public, version=8.5, name=ubi8, build-date=2022-03-28T10:36:18.413762, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=236.1648460182, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.24.2, io.openshift.expose-services=) 2024-05-09T03:57:25.836 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 podman[47725]: 2024-05-09 03:57:25.529171535 +0000 UTC m=+0.407130637 container start 33b2a450dbe16d5881623fba7157c2dc56aa81948aa49aaabdbffd738a1fa12e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, maintainer=Paul Cuzner , name=ubi8, io.openshift.expose-services=, release=236.1648460182, vcs-type=git, com.redhat.component=ubi8-container, distribution-scope=public, io.buildah.version=1.24.2, build-date=2022-03-28T10:36:18.413762, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, architecture=x86_64) 2024-05-09T03:57:25.836 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-05-09T03:57:25.836 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-05-09T03:57:25.836 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-05-09T03:57:25.836 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-05-09T03:57:25.836 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-05-09T03:57:25.836 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-05-09T03:57:25.836 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-05-09T03:57:25.836 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-05-09T03:57:25.836 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="App mode production" logger=settings 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=547.059µs 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-05-09T03:57:25.837 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-05-09T03:57:25.838 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-05-09T03:57:25.838 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-05-09T03:57:25.838 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 bash[47725]: 33b2a450dbe16d5881623fba7157c2dc56aa81948aa49aaabdbffd738a1fa12e 2024-05-09T03:57:25.838 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 systemd[1]: Started Ceph grafana.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T03:57:25.838 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-05-09T03:57:25.838 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-05-09T03:57:25.838 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-05-09T03:57:25.838 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-05-09T03:57:25.838 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-05-09T03:57:25.838 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T03:57:25+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-05-09T03:57:26.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Bus STARTED 2024-05-09T03:57:26.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Bus STOPPING 2024-05-09T03:57:26.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-09T03:57:26.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Bus STOPPED 2024-05-09T03:57:26.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Bus STARTING 2024-05-09T03:57:26.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Serving on http://:::9283 2024-05-09T03:57:26.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Bus STARTED 2024-05-09T03:57:26.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Bus STOPPING 2024-05-09T03:57:26.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-09T03:57:26.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Bus STOPPED 2024-05-09T03:57:26.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:25] ENGINE Bus STARTING 2024-05-09T03:57:26.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:26 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:26] ENGINE Serving on http://:::9283 2024-05-09T03:57:26.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:26 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:03:57:26] ENGINE Bus STARTED 2024-05-09T03:57:26.777 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:26.778 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:26.778 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T03:57:26.778 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T03:57:26.778 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-05-09T03:57:26.778 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-05-09T03:57:26.778 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:26.778 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T03:57:26.778 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T03:57:26.778 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi132.front.sepia.ceph.com:9095"}]: dispatch 2024-05-09T03:57:26.778 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi132.front.sepia.ceph.com:9095"}]: dispatch 2024-05-09T03:57:26.778 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:26.778 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T03:57:26.779 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T03:57:26.779 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi132.front.sepia.ceph.com:3000"}]: dispatch 2024-05-09T03:57:26.779 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi132.front.sepia.ceph.com:3000"}]: dispatch 2024-05-09T03:57:26.779 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:26.779 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:57:26.779 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1214631921' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T03:57:26.779 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1808983332' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3263238082"}]: dispatch 2024-05-09T03:57:26.779 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3263238082"}]: dispatch 2024-05-09T03:57:26.779 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:26 smithi132 ceph-mon[23807]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:57:27.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi132.front.sepia.ceph.com:9095"}]: dispatch 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi132.front.sepia.ceph.com:9095"}]: dispatch 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi132.front.sepia.ceph.com:3000"}]: dispatch 2024-05-09T03:57:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi132.front.sepia.ceph.com:3000"}]: dispatch 2024-05-09T03:57:27.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:27.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:57:27.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1214631921' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T03:57:27.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1808983332' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3263238082"}]: dispatch 2024-05-09T03:57:27.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3263238082"}]: dispatch 2024-05-09T03:57:27.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[27629]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:57:27.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:27.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:27.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T03:57:27.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T03:57:27.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi132.front.sepia.ceph.com:9095"}]: dispatch 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi132.front.sepia.ceph.com:9095"}]: dispatch 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi132.front.sepia.ceph.com:3000"}]: dispatch 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi132.front.sepia.ceph.com:3000"}]: dispatch 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1214631921' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T03:57:27.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1808983332' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3263238082"}]: dispatch 2024-05-09T03:57:27.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3263238082"}]: dispatch 2024-05-09T03:57:27.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:26 smithi078 ceph-mon[22905]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:57:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:27 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3263238082"}]': finished 2024-05-09T03:57:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:27 smithi078 ceph-mon[27629]: osdmap e97: 8 total, 8 up, 8 in 2024-05-09T03:57:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:27 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/408587226' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3227524474"}]: dispatch 2024-05-09T03:57:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:27 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3263238082"}]': finished 2024-05-09T03:57:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:27 smithi078 ceph-mon[22905]: osdmap e97: 8 total, 8 up, 8 in 2024-05-09T03:57:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:27 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/408587226' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3227524474"}]: dispatch 2024-05-09T03:57:28.168 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:27 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3263238082"}]': finished 2024-05-09T03:57:28.168 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:27 smithi132 ceph-mon[23807]: osdmap e97: 8 total, 8 up, 8 in 2024-05-09T03:57:28.168 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:27 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/408587226' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3227524474"}]: dispatch 2024-05-09T03:57:28.996 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:28 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/408587226' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3227524474"}]': finished 2024-05-09T03:57:28.997 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:28 smithi132 ceph-mon[23807]: osdmap e98: 8 total, 8 up, 8 in 2024-05-09T03:57:28.997 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:28 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/266673705' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/330313505"}]: dispatch 2024-05-09T03:57:28.997 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:28 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/330313505"}]: dispatch 2024-05-09T03:57:28.997 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:28 smithi132 ceph-mon[23807]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:57:28.997 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:28.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:57:29.056 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:28 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/408587226' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3227524474"}]': finished 2024-05-09T03:57:29.056 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:28 smithi078 ceph-mon[27629]: osdmap e98: 8 total, 8 up, 8 in 2024-05-09T03:57:29.057 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:28 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/266673705' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/330313505"}]: dispatch 2024-05-09T03:57:29.057 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:28 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/330313505"}]: dispatch 2024-05-09T03:57:29.057 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:28 smithi078 ceph-mon[27629]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:57:29.057 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:28 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/408587226' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3227524474"}]': finished 2024-05-09T03:57:29.057 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:28 smithi078 ceph-mon[22905]: osdmap e98: 8 total, 8 up, 8 in 2024-05-09T03:57:29.057 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:28 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/266673705' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/330313505"}]: dispatch 2024-05-09T03:57:29.057 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:28 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/330313505"}]: dispatch 2024-05-09T03:57:29.057 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:28 smithi078 ceph-mon[22905]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:57:30.331 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:30 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/330313505"}]': finished 2024-05-09T03:57:30.331 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:30 smithi132 ceph-mon[23807]: osdmap e99: 8 total, 8 up, 8 in 2024-05-09T03:57:30.332 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:30 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/992394195' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1359853598"}]: dispatch 2024-05-09T03:57:30.332 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:30 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:30.332 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:30 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:30 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/330313505"}]': finished 2024-05-09T03:57:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:30 smithi078 ceph-mon[22905]: osdmap e99: 8 total, 8 up, 8 in 2024-05-09T03:57:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:30 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/992394195' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1359853598"}]: dispatch 2024-05-09T03:57:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:30 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:30 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:30.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:30 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/330313505"}]': finished 2024-05-09T03:57:30.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:30 smithi078 ceph-mon[27629]: osdmap e99: 8 total, 8 up, 8 in 2024-05-09T03:57:30.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:30 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/992394195' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1359853598"}]: dispatch 2024-05-09T03:57:30.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:30 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:30.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:30 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:31.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/992394195' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1359853598"}]': finished 2024-05-09T03:57:31.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: osdmap e100: 8 total, 8 up, 8 in 2024-05-09T03:57:31.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:31.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:31.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:57:31.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:57:31.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:31.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T03:57:31.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:57:31.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:57:31.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:57:31.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:57:31.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T03:57:31.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:31 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3187565175' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1359853598"}]: dispatch 2024-05-09T03:57:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/992394195' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1359853598"}]': finished 2024-05-09T03:57:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: osdmap e100: 8 total, 8 up, 8 in 2024-05-09T03:57:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:57:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:57:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T03:57:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:57:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:57:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:57:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:57:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T03:57:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3187565175' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1359853598"}]: dispatch 2024-05-09T03:57:31.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/992394195' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/1359853598"}]': finished 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: osdmap e100: 8 total, 8 up, 8 in 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T03:57:31.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:31 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3187565175' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1359853598"}]: dispatch 2024-05-09T03:57:32.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:32 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3187565175' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1359853598"}]': finished 2024-05-09T03:57:32.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:32 smithi078 ceph-mon[22905]: osdmap e101: 8 total, 8 up, 8 in 2024-05-09T03:57:32.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:32 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/826375958' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/460728655"}]: dispatch 2024-05-09T03:57:32.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:32 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/460728655"}]: dispatch 2024-05-09T03:57:32.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:32 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3187565175' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1359853598"}]': finished 2024-05-09T03:57:32.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:32 smithi078 ceph-mon[27629]: osdmap e101: 8 total, 8 up, 8 in 2024-05-09T03:57:32.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:32 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/826375958' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/460728655"}]: dispatch 2024-05-09T03:57:32.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:32 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/460728655"}]: dispatch 2024-05-09T03:57:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:32 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3187565175' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/1359853598"}]': finished 2024-05-09T03:57:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:32 smithi132 ceph-mon[23807]: osdmap e101: 8 total, 8 up, 8 in 2024-05-09T03:57:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:32 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/826375958' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/460728655"}]: dispatch 2024-05-09T03:57:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:32 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/460728655"}]: dispatch 2024-05-09T03:57:33.356 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:33 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/460728655"}]': finished 2024-05-09T03:57:33.356 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:33 smithi078 ceph-mon[22905]: osdmap e102: 8 total, 8 up, 8 in 2024-05-09T03:57:33.356 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:33 smithi078 ceph-mon[22905]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T03:57:33.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:33 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/460728655"}]': finished 2024-05-09T03:57:33.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:33 smithi078 ceph-mon[27629]: osdmap e102: 8 total, 8 up, 8 in 2024-05-09T03:57:33.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:33 smithi078 ceph-mon[27629]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T03:57:33.406 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:33 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/460728655"}]': finished 2024-05-09T03:57:33.406 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:33 smithi132 ceph-mon[23807]: osdmap e102: 8 total, 8 up, 8 in 2024-05-09T03:57:33.406 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:33 smithi132 ceph-mon[23807]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T03:57:33.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi132\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi132\", job=\"node\", machine=\"x86_64\", nodename=\"smithi132\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi132\", job=\"node\", machine=\"x86_64\", nodename=\"smithi132\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:57:33.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T03:57:33.426+0000 7f4ffcefb700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:33.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Creating ceph-iscsi config... 2024-05-09T03:57:33.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:33.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:33.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:33.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:33.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:33.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T03:57:33.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:33.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return _run_code(code, main_globals, None, 2024-05-09T03:57:33.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:33.756 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: exec(code, run_globals) 2024-05-09T03:57:33.756 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:33.756 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:33.756 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:33.756 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:33.756 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:33.756 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:33.756 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:33.756 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return future.result(timeout) 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return self.__get_result() 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:57:33.757 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: raise self._exception 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: use_current_daemon_image=reconfig, 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Creating ceph-iscsi config... 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:33.758 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return _run_code(code, main_globals, None, 2024-05-09T03:57:33.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:33.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: exec(code, run_globals) 2024-05-09T03:57:33.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:33.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:33.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:33.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:33.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:33.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:33.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:33.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:33.759 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:34.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:34 smithi132 ceph-mon[23807]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:34.504 INFO:journalctl@ceph.mon.b.smithi132.stdout: Creating ceph-iscsi config... 2024-05-09T03:57:34.504 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: exec(code, run_globals) 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:34.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: return future.result(timeout) 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:57:34.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: return self.__get_result() 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: raise self._exception 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: use_current_daemon_image=reconfig, 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: Creating ceph-iscsi config... 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:34.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:57:34.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:34.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: exec(code, run_globals) 2024-05-09T03:57:34.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:34.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:34.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:34.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:34.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:34.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:34.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:34.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:34.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:34.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:34 smithi078 ceph-mon[22905]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:34.512 INFO:journalctl@ceph.mon.a.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:34.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: return future.result(timeout) 2024-05-09T03:57:34.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: return self.__get_result() 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: raise self._exception 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: use_current_daemon_image=reconfig, 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:34.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:34.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:34 smithi078 ceph-mon[27629]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:34.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:57:34.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: return future.result(timeout) 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: return self.__get_result() 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: raise self._exception 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: use_current_daemon_image=reconfig, 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:57:34.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:57:34.520 INFO:journalctl@ceph.mon.c.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:57:34.521 INFO:journalctl@ceph.mon.c.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:57:35.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:35 smithi132 ceph-mon[23807]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 1 op/s 2024-05-09T03:57:35.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:57:34] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-05-09T03:57:35.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:35 smithi078 ceph-mon[22905]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 1 op/s 2024-05-09T03:57:35.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:35 smithi078 ceph-mon[27629]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 1 op/s 2024-05-09T03:57:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:37 smithi132 ceph-mon[23807]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 808 B/s rd, 0 op/s 2024-05-09T03:57:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:37 smithi078 ceph-mon[22905]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 808 B/s rd, 0 op/s 2024-05-09T03:57:37.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:37 smithi078 ceph-mon[27629]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 808 B/s rd, 0 op/s 2024-05-09T03:57:38.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:38 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:57:38.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:57:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:38 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:57:38.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:38 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:57:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:39 smithi132 ceph-mon[23807]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-09T03:57:39.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:39 smithi078 ceph-mon[22905]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-09T03:57:39.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:39 smithi078 ceph-mon[27629]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-09T03:57:41.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:41 smithi132 ceph-mon[23807]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 550 B/s rd, 0 op/s 2024-05-09T03:57:41.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:41 smithi078 ceph-mon[22905]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 550 B/s rd, 0 op/s 2024-05-09T03:57:41.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:41 smithi078 ceph-mon[27629]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 550 B/s rd, 0 op/s 2024-05-09T03:57:43.753 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:43 smithi078 ceph-mon[22905]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 497 B/s rd, 0 op/s 2024-05-09T03:57:43.754 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:43 smithi078 ceph-mon[27629]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 497 B/s rd, 0 op/s 2024-05-09T03:57:43.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:43 smithi132 ceph-mon[23807]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 497 B/s rd, 0 op/s 2024-05-09T03:57:43.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:57:45.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:44 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:57:44] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-05-09T03:57:45.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:45 smithi132 ceph-mon[23807]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-09T03:57:45.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:45 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3126852342' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T03:57:45.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:45 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/593330660' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/477924627"}]: dispatch 2024-05-09T03:57:45.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:45 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/477924627"}]: dispatch 2024-05-09T03:57:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:45 smithi078 ceph-mon[22905]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-09T03:57:45.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:45 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3126852342' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T03:57:45.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:45 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/593330660' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/477924627"}]: dispatch 2024-05-09T03:57:45.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:45 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/477924627"}]: dispatch 2024-05-09T03:57:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:45 smithi078 ceph-mon[27629]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-09T03:57:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:45 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3126852342' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T03:57:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:45 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/593330660' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/477924627"}]: dispatch 2024-05-09T03:57:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:45 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/477924627"}]: dispatch 2024-05-09T03:57:46.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:46 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/477924627"}]': finished 2024-05-09T03:57:46.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:46 smithi132 ceph-mon[23807]: osdmap e103: 8 total, 8 up, 8 in 2024-05-09T03:57:46.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:46 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/477924627"}]': finished 2024-05-09T03:57:46.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:46 smithi078 ceph-mon[22905]: osdmap e103: 8 total, 8 up, 8 in 2024-05-09T03:57:46.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:46 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/477924627"}]': finished 2024-05-09T03:57:46.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:46 smithi078 ceph-mon[27629]: osdmap e103: 8 total, 8 up, 8 in 2024-05-09T03:57:47.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:47 smithi132 ceph-mon[23807]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:57:47.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:47 smithi078 ceph-mon[22905]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:57:47.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:47 smithi078 ceph-mon[27629]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:57:49.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:48 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:57:49.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:49 smithi132 ceph-mon[23807]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:57:49.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:49 smithi078 ceph-mon[22905]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:57:49.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:49 smithi078 ceph-mon[27629]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-09T03:57:51.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:51 smithi132 ceph-mon[23807]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-05-09T03:57:51.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:51 smithi078 ceph-mon[22905]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-05-09T03:57:51.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:51 smithi078 ceph-mon[27629]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-05-09T03:57:53.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:57:53.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:53 smithi132 ceph-mon[23807]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-05-09T03:57:53.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:57:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:53 smithi078 ceph-mon[22905]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-05-09T03:57:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:57:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:53 smithi078 ceph-mon[27629]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-05-09T03:57:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:57:55.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:57:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:57:54] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-05-09T03:57:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:55 smithi132 ceph-mon[23807]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-09T03:57:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:55 smithi078 ceph-mon[22905]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-09T03:57:55.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:55 smithi078 ceph-mon[27629]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-09T03:57:56.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:56 smithi132 ceph-mon[23807]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:57:56.904 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:56 smithi078 ceph-mon[22905]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:57:56.905 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:56 smithi078 ceph-mon[27629]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:57:57.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:57 smithi132 ceph-mon[23807]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-05-09T03:57:57.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:57 smithi078 ceph-mon[27629]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-05-09T03:57:57.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:57 smithi078 ceph-mon[22905]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 843 B/s rd, 0 op/s 2024-05-09T03:57:59.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:57:58 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:57:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:57:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:57:59 smithi132 ceph-mon[23807]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T03:57:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:57:59 smithi078 ceph-mon[22905]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T03:57:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:57:59 smithi078 ceph-mon[27629]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T03:58:01.761 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:01 smithi132 ceph-mon[23807]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:01.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:01 smithi078 ceph-mon[22905]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:01.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:01 smithi078 ceph-mon[27629]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:03.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:03 smithi132 ceph-mon[23807]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:03.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:58:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:58:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:58:03.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:03 smithi078 ceph-mon[22905]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:03.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:03 smithi078 ceph-mon[27629]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:05.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:04 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:58:04] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-05-09T03:58:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:05 smithi132 ceph-mon[23807]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:05.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:05 smithi078 ceph-mon[27629]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:05.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:05 smithi078 ceph-mon[22905]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:06.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:06 smithi132 ceph-mon[23807]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:58:06.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:06 smithi078 ceph-mon[22905]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:58:06.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:06 smithi078 ceph-mon[27629]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:58:07.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:07 smithi132 ceph-mon[23807]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:07.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:07 smithi078 ceph-mon[22905]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:07.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:07 smithi078 ceph-mon[27629]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:08 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:58:08.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:58:08 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:58:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:58:08.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:08 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:58:08.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:08 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:58:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:09 smithi132 ceph-mon[23807]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:09 smithi078 ceph-mon[22905]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:09.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:09 smithi078 ceph-mon[27629]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:11 smithi132 ceph-mon[23807]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:11.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:11 smithi078 ceph-mon[22905]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:11 smithi078 ceph-mon[27629]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:13 smithi132 ceph-mon[23807]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:13.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:58:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:58:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:58:13.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:13 smithi078 ceph-mon[22905]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:13 smithi078 ceph-mon[27629]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:15.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:58:14] "GET /metrics HTTP/1.1" 200 37971 "" "Prometheus/2.33.4" 2024-05-09T03:58:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:15 smithi132 ceph-mon[23807]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:15 smithi078 ceph-mon[22905]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:15.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:15 smithi078 ceph-mon[27629]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:16 smithi132 ceph-mon[23807]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:58:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:16 smithi078 ceph-mon[22905]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:58:16.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:16 smithi078 ceph-mon[27629]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:58:17.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:17 smithi132 ceph-mon[23807]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:17.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:17 smithi078 ceph-mon[22905]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:17.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:17 smithi078 ceph-mon[27629]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:19.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:58:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:58:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:58:19.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:19 smithi132 ceph-mon[23807]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:19.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:19 smithi078 ceph-mon[22905]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:19.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:19 smithi078 ceph-mon[27629]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:21.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:21 smithi132 ceph-mon[23807]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:21.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:21 smithi078 ceph-mon[22905]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:21.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:21 smithi078 ceph-mon[27629]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:23.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:58:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:58:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:58:23.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:23 smithi132 ceph-mon[23807]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:23.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:23 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:58:23.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:23 smithi078 ceph-mon[22905]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:23.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:23 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:58:23.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:23 smithi078 ceph-mon[27629]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:23.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:23 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:58:25.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:58:24] "GET /metrics HTTP/1.1" 200 37971 "" "Prometheus/2.33.4" 2024-05-09T03:58:25.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:25 smithi132 ceph-mon[23807]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:25 smithi078 ceph-mon[22905]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:25 smithi078 ceph-mon[27629]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:26 smithi132 ceph-mon[23807]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:58:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:26 smithi078 ceph-mon[22905]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:58:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:26 smithi078 ceph-mon[27629]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:58:27.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:27 smithi132 ceph-mon[23807]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:27.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:27 smithi078 ceph-mon[22905]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:27.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:27 smithi078 ceph-mon[27629]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:29.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:58:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:58:28.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:58:29.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:29 smithi132 ceph-mon[23807]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:29.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:29 smithi078 ceph-mon[22905]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:29.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:29 smithi078 ceph-mon[27629]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:31 smithi132 ceph-mon[23807]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:31.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:31 smithi078 ceph-mon[22905]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:31.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:31 smithi078 ceph-mon[27629]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:33.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:33 smithi132 ceph-mon[23807]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:33.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:33 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:58:33.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:58:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:58:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:58:33.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:33 smithi078 ceph-mon[27629]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:33.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:33 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:58:33.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:33 smithi078 ceph-mon[22905]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:33.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:33 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:58:35.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:58:34] "GET /metrics HTTP/1.1" 200 37971 "" "Prometheus/2.33.4" 2024-05-09T03:58:35.752 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:35 smithi132 ceph-mon[23807]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:35.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:35 smithi078 ceph-mon[22905]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:35.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:35 smithi078 ceph-mon[27629]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:36.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:36 smithi132 ceph-mon[23807]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:58:36.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:36 smithi078 ceph-mon[22905]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:58:36.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:36 smithi078 ceph-mon[27629]: from='client.15150 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:58:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:37 smithi132 ceph-mon[23807]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:37 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:37 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:37 smithi078 ceph-mon[22905]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:37 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:37 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:37.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:37 smithi078 ceph-mon[27629]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:37.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:37 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:37.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:37 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:38.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:58:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:58:38.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:58:38.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:38 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:58:38.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:38 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:38.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:38 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:38.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:38 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:58:38.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:38 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:58:38.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:38 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:38.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:38 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:58:38.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:38 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:58:38.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:38 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:58:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:58:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:58:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:58:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:58:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:58:38.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:58:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:58:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:58:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:58:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:58:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:58:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:58:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:38 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:58:39.758 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:39 smithi132 ceph-mon[23807]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T03:58:39.758 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:39 smithi132 ceph-mon[23807]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:58:39.758 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:39 smithi132 ceph-mon[23807]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:39.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:39 smithi078 ceph-mon[22905]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T03:58:39.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:39 smithi078 ceph-mon[22905]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:58:39.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:39 smithi078 ceph-mon[22905]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:39.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:39 smithi078 ceph-mon[27629]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T03:58:39.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:39 smithi078 ceph-mon[27629]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:58:39.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:39 smithi078 ceph-mon[27629]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:41.721 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:41 smithi078 ceph-mon[22905]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:41.721 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:41 smithi078 ceph-mon[27629]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:41.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:41 smithi132 ceph-mon[23807]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:58:42.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T03:58:41.797+0000 7f4ffcefb700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:58:42.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Creating ceph-iscsi config... 2024-05-09T03:58:42.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:58:42.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:58:42.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:58:42.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:42.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:42.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T03:58:42.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:58:42.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return _run_code(code, main_globals, None, 2024-05-09T03:58:42.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:58:42.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: exec(code, run_globals) 2024-05-09T03:58:42.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:58:42.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:58:42.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:58:42.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:58:42.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:58:42.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:58:42.256 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:58:42.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:42.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:42.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T03:58:42.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:58:42.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:58:42.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:58:42.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:58:42.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:58:42.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:58:42.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:58:42.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return future.result(timeout) 2024-05-09T03:58:42.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:58:42.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return self.__get_result() 2024-05-09T03:58:42.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:58:42.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: raise self._exception 2024-05-09T03:58:42.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:58:42.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: use_current_daemon_image=reconfig, 2024-05-09T03:58:42.258 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Creating ceph-iscsi config... 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return _run_code(code, main_globals, None, 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: exec(code, run_globals) 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:58:42.259 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:58:42.260 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:58:42.260 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:58:42.260 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:58:42.260 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:58:42.260 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:42.260 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:41 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:58:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:58:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:58:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:43 smithi132 ceph-mon[23807]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:58:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Creating ceph-iscsi config... 2024-05-09T03:58:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:58:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:58:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:58:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T03:58:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:58:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:58:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: exec(code, run_globals) 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:58:43.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: return future.result(timeout) 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: return self.__get_result() 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: raise self._exception 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: use_current_daemon_image=reconfig, 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Creating ceph-iscsi config... 2024-05-09T03:58:43.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: exec(code, run_globals) 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:58:43.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:58:43.759 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:58:43.759 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:58:43.759 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:58:43.759 INFO:journalctl@ceph.mon.b.smithi132.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:43.759 INFO:journalctl@ceph.mon.b.smithi132.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.759 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:43 smithi132 ceph-mon[23807]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:43.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:43 smithi078 ceph-mon[27629]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:58:43.762 INFO:journalctl@ceph.mon.c.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:58:43.762 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:58:43.762 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:58:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: return future.result(timeout) 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: return self.__get_result() 2024-05-09T03:58:43.764 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: raise self._exception 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: use_current_daemon_image=reconfig, 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:58:43.765 INFO:journalctl@ceph.mon.c.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:58:43.766 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:58:43.766 INFO:journalctl@ceph.mon.c.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:58:43.766 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:58:43.766 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:58:43.766 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:58:43.766 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:58:43.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:43 smithi078 ceph-mon[22905]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:58:43.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:58:43.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:58:43.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:58:43.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:58:43.767 INFO:journalctl@ceph.mon.a.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:43.768 INFO:journalctl@ceph.mon.a.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.768 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:58:43.768 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:58:43.768 INFO:journalctl@ceph.mon.a.smithi078.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:58:43.768 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:58:43.768 INFO:journalctl@ceph.mon.a.smithi078.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:58:43.768 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:58:43.768 INFO:journalctl@ceph.mon.a.smithi078.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:58:43.768 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:58:43.768 INFO:journalctl@ceph.mon.a.smithi078.stdout: return future.result(timeout) 2024-05-09T03:58:43.768 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:58:43.768 INFO:journalctl@ceph.mon.a.smithi078.stdout: return self.__get_result() 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: raise self._exception 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: use_current_daemon_image=reconfig, 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:58:43.769 INFO:journalctl@ceph.mon.a.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:58:43.770 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:58:43.770 INFO:journalctl@ceph.mon.a.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:58:43.770 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:58:43.770 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:58:43.770 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:58:43.770 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:58:43.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:58:43.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:58:43.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:58:43.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:43.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:43 smithi078 ceph-mon[27629]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:43.771 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:58:43.771 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:58:43.771 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:58:43.771 INFO:journalctl@ceph.mon.a.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:58:43.771 INFO:journalctl@ceph.mon.a.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:58:43.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:43 smithi078 ceph-mon[22905]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:58:45.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:44 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:58:44] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-05-09T03:58:45.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:45 smithi132 ceph-mon[23807]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-09T03:58:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:45 smithi078 ceph-mon[22905]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-09T03:58:45.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:45 smithi078 ceph-mon[27629]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-09T03:58:47.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:47 smithi132 ceph-mon[23807]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-09T03:58:47.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:47 smithi078 ceph-mon[22905]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-09T03:58:47.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:47 smithi078 ceph-mon[27629]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-09T03:58:49.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:58:48 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:58:48.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:58:49.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:49 smithi078 ceph-mon[22905]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-09T03:58:49.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:49 smithi078 ceph-mon[27629]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-09T03:58:49.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:49 smithi132 ceph-mon[23807]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-09T03:58:51.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:51 smithi132 ceph-mon[23807]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-09T03:58:51.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:51 smithi078 ceph-mon[22905]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-09T03:58:51.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:51 smithi078 ceph-mon[27629]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-09T03:58:53.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:58:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:58:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:58:53.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:53 smithi132 ceph-mon[23807]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-09T03:58:53.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:58:53.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:53 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1649853994' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T03:58:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:53 smithi078 ceph-mon[27629]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-09T03:58:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:58:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:53 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1649853994' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T03:58:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:53 smithi078 ceph-mon[22905]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-09T03:58:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:58:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:53 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1649853994' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T03:58:55.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:58:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:58:54] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-05-09T03:58:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:55 smithi132 ceph-mon[23807]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-09T03:58:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:55 smithi078 ceph-mon[22905]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-09T03:58:55.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:55 smithi078 ceph-mon[27629]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-09T03:58:57.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:57 smithi132 ceph-mon[23807]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-09T03:58:57.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:57 smithi078 ceph-mon[22905]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-09T03:58:57.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:57 smithi078 ceph-mon[27629]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-09T03:58:59.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:58:58 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:58:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:58:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:58:59 smithi132 ceph-mon[23807]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:58:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:58:59 smithi078 ceph-mon[22905]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:58:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:58:59 smithi078 ceph-mon[27629]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:59:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:01 smithi132 ceph-mon[23807]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:59:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:01 smithi078 ceph-mon[22905]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:59:01.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:01 smithi078 ceph-mon[27629]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:59:03.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:59:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:59:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:59:03.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:03 smithi132 ceph-mon[23807]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:59:03.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:03 smithi078 ceph-mon[22905]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:59:03.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:03 smithi078 ceph-mon[27629]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:59:04.716 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:04 smithi132 ceph-mon[23807]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:04.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:04 smithi078 ceph-mon[22905]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:04.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:04 smithi078 ceph-mon[27629]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:05.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:04 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:59:04] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-05-09T03:59:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:05 smithi132 ceph-mon[23807]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:59:05.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:05 smithi078 ceph-mon[22905]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:59:05.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:05 smithi078 ceph-mon[27629]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:59:07.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:07 smithi132 ceph-mon[23807]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:07.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:07 smithi078 ceph-mon[22905]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:07.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:07 smithi078 ceph-mon[27629]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:08 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:59:08.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:59:08 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:59:08.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:59:08.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:08 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:59:08.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:08 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:59:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:09 smithi132 ceph-mon[23807]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:09 smithi078 ceph-mon[22905]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:09.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:09 smithi078 ceph-mon[27629]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:11 smithi132 ceph-mon[23807]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:11.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:11 smithi078 ceph-mon[22905]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:11 smithi078 ceph-mon[27629]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:13 smithi132 ceph-mon[23807]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:13.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:59:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:59:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:59:13.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:13 smithi078 ceph-mon[22905]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:13 smithi078 ceph-mon[27629]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:14.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:14 smithi132 ceph-mon[23807]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:14.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:14 smithi078 ceph-mon[22905]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:14.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:14 smithi078 ceph-mon[27629]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:15.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:59:14] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-05-09T03:59:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:15 smithi132 ceph-mon[23807]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:15 smithi078 ceph-mon[22905]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:15.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:15 smithi078 ceph-mon[27629]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:17.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:17 smithi132 ceph-mon[23807]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:17.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:17 smithi078 ceph-mon[22905]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:17.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:17 smithi078 ceph-mon[27629]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:19.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:59:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:59:18.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:59:19.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:19 smithi132 ceph-mon[23807]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:19.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:19 smithi078 ceph-mon[22905]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:19.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:19 smithi078 ceph-mon[27629]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:21.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:21 smithi132 ceph-mon[23807]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:21.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:21 smithi078 ceph-mon[22905]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:21.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:21 smithi078 ceph-mon[27629]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:23.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:23 smithi132 ceph-mon[23807]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:23.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:23 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:59:23.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:59:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:59:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:59:23.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:23 smithi078 ceph-mon[22905]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:23.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:23 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:59:23.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:23 smithi078 ceph-mon[27629]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:23.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:23 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:59:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:24 smithi132 ceph-mon[23807]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:24 smithi078 ceph-mon[22905]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:24 smithi078 ceph-mon[27629]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:25.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:59:24] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-05-09T03:59:25.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:25 smithi132 ceph-mon[23807]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:25 smithi078 ceph-mon[22905]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:25 smithi078 ceph-mon[27629]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:27.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:27 smithi132 ceph-mon[23807]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:27.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:27 smithi078 ceph-mon[27629]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:27.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:27 smithi078 ceph-mon[22905]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:29.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:59:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:59:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:59:29.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:29 smithi132 ceph-mon[23807]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:29.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:29 smithi078 ceph-mon[22905]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:29.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:29 smithi078 ceph-mon[27629]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:31 smithi132 ceph-mon[23807]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:31.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:31 smithi078 ceph-mon[22905]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:31.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:31 smithi078 ceph-mon[27629]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:33.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:33 smithi132 ceph-mon[23807]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:33.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:59:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:59:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:59:33.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:33 smithi078 ceph-mon[22905]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:33.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:33 smithi078 ceph-mon[27629]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:34.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:34 smithi132 ceph-mon[23807]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:34.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:34 smithi078 ceph-mon[22905]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:34.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:34 smithi078 ceph-mon[27629]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:35.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:59:34] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-05-09T03:59:35.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:35 smithi132 ceph-mon[23807]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:35.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:35 smithi078 ceph-mon[22905]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:35.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:35 smithi078 ceph-mon[27629]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:37 smithi132 ceph-mon[23807]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:37 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:59:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:37 smithi078 ceph-mon[22905]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:37 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:59:37.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:37 smithi078 ceph-mon[27629]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:37.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:37 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:59:39.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:59:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:59:38.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:59:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:39 smithi132 ceph-mon[23807]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:39.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:39 smithi078 ceph-mon[22905]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:39.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:39 smithi078 ceph-mon[27629]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:41.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:41 smithi132 ceph-mon[23807]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:41.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:41 smithi078 ceph-mon[22905]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:41.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:41 smithi078 ceph-mon[27629]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:42.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:42 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:59:42.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:42 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:59:42.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:42 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:59:43.623 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:43 smithi132 ceph-mon[23807]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:43.623 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:59:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:59:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:59:43.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:43 smithi078 ceph-mon[22905]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:43.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:43 smithi078 ceph-mon[27629]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:44.577 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:44 smithi132 ceph-mon[23807]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:44.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:44 smithi078 ceph-mon[22905]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:44.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:44 smithi078 ceph-mon[27629]: from='client.25072 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T03:59:45.200 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:44 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:59:44] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-05-09T03:59:45.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:45 smithi132 ceph-mon[23807]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:45 smithi078 ceph-mon[22905]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:45.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:45 smithi078 ceph-mon[27629]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:47.572 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:47.572 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:47.572 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[22905]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:47.572 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:59:47.572 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:59:47.572 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:47.573 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:59:47.573 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:59:47.573 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:59:47.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:47.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:47.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[27629]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:47.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:59:47.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:59:47.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:47.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:59:47.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:59:47.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:47 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:59:47.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:47 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:47.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:47 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:47.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:47 smithi132 ceph-mon[23807]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:47.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:47 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:59:47.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:47 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T03:59:47.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:47 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:47.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:47 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:59:47.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:47 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T03:59:47.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:47 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T03:59:48.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:48 smithi078 ceph-mon[27629]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T03:59:48.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:48 smithi078 ceph-mon[27629]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:59:48.604 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:48 smithi078 ceph-mon[22905]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T03:59:48.604 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:48 smithi078 ceph-mon[22905]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:59:48.660 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:48 smithi132 ceph-mon[23807]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T03:59:48.660 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:48 smithi132 ceph-mon[23807]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T03:59:49.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:59:48 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:59:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:59:49.507 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:49 smithi078 ceph-mon[27629]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:49.507 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:49 smithi078 ceph-mon[22905]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:49.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:49 smithi132 ceph-mon[23807]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T03:59:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T03:59:50.217+0000 7f4ffcefb700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:59:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Creating ceph-iscsi config... 2024-05-09T03:59:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:59:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:59:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:59:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T03:59:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:59:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return _run_code(code, main_globals, None, 2024-05-09T03:59:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:59:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: exec(code, run_globals) 2024-05-09T03:59:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:59:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:59:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:59:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:59:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:59:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:59:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:59:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T03:59:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:59:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:59:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:59:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:59:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return future.result(timeout) 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return self.__get_result() 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: raise self._exception 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: use_current_daemon_image=reconfig, 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Creating ceph-iscsi config... 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:59:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return _run_code(code, main_globals, None, 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: exec(code, run_globals) 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:59:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:59:50.510 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:59:50.510 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:59:50.510 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:50.510 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:51 smithi132 ceph-mon[23807]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:59:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Creating ceph-iscsi config... 2024-05-09T03:59:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:59:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:59:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:59:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T03:59:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:59:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:59:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:59:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: exec(code, run_globals) 2024-05-09T03:59:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:59:51.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: return future.result(timeout) 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: return self.__get_result() 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: raise self._exception 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: use_current_daemon_image=reconfig, 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Creating ceph-iscsi config... 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:59:51.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: exec(code, run_globals) 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:59:51.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:59:51.759 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:59:51.759 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:59:51.759 INFO:journalctl@ceph.mon.b.smithi132.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:51.759 INFO:journalctl@ceph.mon.b.smithi132.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.759 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:51 smithi132 ceph-mon[23807]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:51.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:51 smithi078 ceph-mon[22905]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:59:51.762 INFO:journalctl@ceph.mon.a.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:59:51.762 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:59:51.762 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:59:51.762 INFO:journalctl@ceph.mon.a.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:59:51.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: return future.result(timeout) 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: return self.__get_result() 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:59:51.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: raise self._exception 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: use_current_daemon_image=reconfig, 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:59:51.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:59:51.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:59:51.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:59:51.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:59:51.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:59:51.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:59:51.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:51 smithi078 ceph-mon[27629]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:59:51.766 INFO:journalctl@ceph.mon.c.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:59:51.766 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:59:51.766 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:59:51.766 INFO:journalctl@ceph.mon.c.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:59:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: return future.result(timeout) 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: return self.__get_result() 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: raise self._exception 2024-05-09T03:59:51.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: use_current_daemon_image=reconfig, 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T03:59:51.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: exec(code, run_globals) 2024-05-09T03:59:51.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T03:59:51.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T03:59:51.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T03:59:51.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T03:59:51.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:59:51.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:59:51.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:59:51.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:51.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.770 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:51 smithi078 ceph-mon[27629]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:51.770 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T03:59:51.771 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T03:59:51.771 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T03:59:51.771 INFO:journalctl@ceph.mon.a.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T03:59:51.771 INFO:journalctl@ceph.mon.a.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T03:59:51.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:51 smithi078 ceph-mon[22905]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:53.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:53 smithi132 ceph-mon[23807]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:53.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:59:53.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:59:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:59:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:59:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:53 smithi078 ceph-mon[22905]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:59:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:53 smithi078 ceph-mon[27629]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T03:59:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T03:59:55.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 03:59:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:03:59:54] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-05-09T03:59:55.444 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-09T03:59:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:55 smithi132 ceph-mon[23807]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:59:55.756 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:55 smithi078 ceph-mon[22905]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:59:55.756 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:55 smithi078 ceph-mon[27629]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-09T03:59:57.676 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:57 smithi078 ceph-mon[22905]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:59:57.677 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:57 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3385499615' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-09T03:59:57.677 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:57 smithi078 ceph-mon[27629]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:59:57.677 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:57 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3385499615' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-09T03:59:57.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:57 smithi132 ceph-mon[23807]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:59:57.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:57 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3385499615' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-09T03:59:57.865 INFO:teuthology.orchestra.run.smithi078.stdout:Scheduled to redeploy mgr.y on host 'smithi078' 2024-05-09T03:59:58.562 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps --refresh' 2024-05-09T03:59:59.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 03:59:58 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T03:59:58.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T03:59:59.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:58 smithi132 ceph-mon[23807]: from='client.25093 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:59:59.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:58 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:59.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:58 smithi132 ceph-mon[23807]: Schedule redeploy daemon mgr.y 2024-05-09T03:59:59.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:58 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:59.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:58 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:59.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:58 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:59:59.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 03:59:58 smithi132 ceph-mon[23807]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:59:59.124 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[22905]: from='client.25093 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:59:59.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:59.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[22905]: Schedule redeploy daemon mgr.y 2024-05-09T03:59:59.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:59.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:59.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:59:59.125 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[22905]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T03:59:59.125 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[27629]: from='client.25093 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T03:59:59.125 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:59.126 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[27629]: Schedule redeploy daemon mgr.y 2024-05-09T03:59:59.126 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:59.126 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T03:59:59.126 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T03:59:59.126 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 03:59:58 smithi078 ceph-mon[27629]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:00:00.201 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:00 smithi078 ceph-mon[22905]: overall HEALTH_OK 2024-05-09T04:00:00.201 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:00 smithi078 ceph-mon[27629]: overall HEALTH_OK 2024-05-09T04:00:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:00 smithi132 ceph-mon[23807]: overall HEALTH_OK 2024-05-09T04:00:00.705 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:00:00.706 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (2m) 83s ago 10m 16.0M - 0.23.0 ba2b418f427c f44cb50f5f35 2024-05-09T04:00:00.706 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (2m) 14s ago 10m 44.3M - 8.3.5 dad864ee21e9 33b2a450dbe1 2024-05-09T04:00:00.706 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (2m) 83s ago 9m 48.4M - 3.5 e1d6a67b021e 0174d5f51c91 2024-05-09T04:00:00.706 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283 running (5m) 14s ago 15m 522M - 19.0.0-3631-gc66c553a af4a4df59e0a 4a3f6038cf25 2024-05-09T04:00:00.706 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:9283 running (16m) 83s ago 16m 410M - 17.2.0 e1d6a67b021e b2a9d11f22f2 2024-05-09T04:00:00.706 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (16m) 83s ago 17m 55.7M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T04:00:00.706 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (15m) 14s ago 15m 50.2M 2048M 17.2.0 e1d6a67b021e a71bae52559d 2024-05-09T04:00:00.706 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (15m) 83s ago 15m 49.0M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T04:00:00.706 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (2m) 83s ago 11m 16.6M - 1.3.1 1dbe0e931976 4a3cc1414851 2024-05-09T04:00:00.706 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (2m) 14s ago 11m 17.0M - 1.3.1 1dbe0e931976 666630a53114 2024-05-09T04:00:00.706 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (14m) 83s ago 14m 52.1M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:00:00.707 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (14m) 83s ago 14m 55.3M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:00:00.707 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (14m) 83s ago 14m 49.1M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:00:00.707 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (13m) 83s ago 13m 51.5M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:00:00.707 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (13m) 14s ago 13m 53.4M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:00:00.707 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (12m) 14s ago 12m 50.7M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:00:00.707 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (12m) 14s ago 12m 50.6M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:00:00.707 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (12m) 14s ago 12m 53.9M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:00:00.707 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (2m) 14s ago 11m 70.9M - 2.33.4 514e6a882f6e 860656079d4c 2024-05-09T04:00:00.707 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (9m) 83s ago 9m 73.9M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:00:00.707 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (9m) 14s ago 9m 75.3M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:00:01.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:01 smithi078 ceph-mon[27629]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-09T04:00:01.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:01 smithi078 ceph-mon[22905]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-09T04:00:01.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:01 smithi132 ceph-mon[23807]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-09T04:00:02.132 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'sleep 180' 2024-05-09T04:00:02.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:02 smithi132 ceph-mon[23807]: from='client.15207 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:00:02.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:02 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:02.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:02 smithi078 ceph-mon[22905]: from='client.15207 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:00:02.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:02 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:02.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:02 smithi078 ceph-mon[27629]: from='client.15207 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:00:02.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:02 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:03.562 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:00:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:00:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:00:03.870 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:03 smithi078 ceph-mon[22905]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-09T04:00:03.870 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:03 smithi078 ceph-mon[27629]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-09T04:00:04.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:03 smithi132 ceph-mon[23807]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-09T04:00:04.716 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:04 smithi078 ceph-mon[27629]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-09T04:00:04.716 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:04 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3661561184' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:00:04.716 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:04 smithi078 ceph-mon[22905]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-09T04:00:04.717 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:04 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3661561184' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:00:04.888 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:04 smithi132 ceph-mon[23807]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-09T04:00:04.888 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:04 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3661561184' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:00:05.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:04 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:00:04] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-05-09T04:00:07.163 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:06 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:07.163 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:06 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:07.163 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:06 smithi078 ceph-mon[27629]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:00:07.163 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:06 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:07.164 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:06 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:07.164 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:06 smithi078 ceph-mon[22905]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:00:07.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:06 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:07.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:06 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:07.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:06 smithi132 ceph-mon[23807]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:00:08.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:08.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:08.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:08.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:00:08.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:08.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:00:08.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:00:08.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:00:08.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T04:00:08.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:08.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[27629]: Deploying daemon mgr.y on smithi078 2024-05-09T04:00:08.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:08.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:08.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:08.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:00:08.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:08.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:00:08.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:00:08.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:00:08.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T04:00:08.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:08.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:08 smithi078 ceph-mon[22905]: Deploying daemon mgr.y on smithi078 2024-05-09T04:00:08.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:00:08 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:00:08.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:00:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:08 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:08 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:08 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:08 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:00:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:08 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:08 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:00:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:08 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:00:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:08 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:00:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:08 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T04:00:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:08 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:08.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:08 smithi132 ceph-mon[23807]: Deploying daemon mgr.y on smithi078 2024-05-09T04:00:09.674 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:09 smithi078 ceph-mon[22905]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:00:09.675 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:09 smithi078 ceph-mon[27629]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:00:09.675 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:09 smithi078 systemd[1]: Stopping Ceph mgr.y for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:00:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:09 smithi132 ceph-mon[23807]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:00:09.979 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:09 smithi078 podman[61569]: 2024-05-09 04:00:09.673349902 +0000 UTC m=+0.141438768 container died b2a9d11f22f2540da5bf9578f4559da1ec730c6e98fad3fec6a7a20846afe592 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.buildah.version=1.19.8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, version=8, io.openshift.expose-services=, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True) 2024-05-09T04:00:10.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:10 smithi078 podman[61569]: 2024-05-09 04:00:10.06912575 +0000 UTC m=+0.537214614 container cleanup b2a9d11f22f2540da5bf9578f4559da1ec730c6e98fad3fec6a7a20846afe592 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, architecture=x86_64, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, RELEASE=HEAD, name=centos-stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8) 2024-05-09T04:00:10.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:10 smithi078 bash[61569]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y 2024-05-09T04:00:10.616 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:10 smithi078 podman[61583]: 2024-05-09 04:00:10.287444081 +0000 UTC m=+0.618387894 container remove b2a9d11f22f2540da5bf9578f4559da1ec730c6e98fad3fec6a7a20846afe592 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, release=754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., distribution-scope=public, vcs-type=git, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, version=8, CEPH_POINT_RELEASE=-17.2.0) 2024-05-09T04:00:10.617 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:10 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-05-09T04:00:10.617 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:10 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-05-09T04:00:10.617 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:10 smithi078 systemd[1]: Stopped Ceph mgr.y for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:00:10.617 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:10 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.y.service: Consumed 1min 392ms CPU time. 2024-05-09T04:00:10.958 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:10 smithi078 systemd[1]: Starting Ceph mgr.y for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:00:10.958 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:10 smithi078 podman[61718]: 2024-05-09 04:00:10.830824215 +0000 UTC m=+0.100186498 container create cc8cc8e980a83cf6112186e033829bf618c9e2bf96fd00abcb9f8c1398b0c2c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507) 2024-05-09T04:00:10.958 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:10 smithi078 podman[61718]: 2024-05-09 04:00:10.749800832 +0000 UTC m=+0.019163136 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:00:11.430 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:11 smithi078 podman[61718]: 2024-05-09 04:00:11.109258399 +0000 UTC m=+0.378620683 container init cc8cc8e980a83cf6112186e033829bf618c9e2bf96fd00abcb9f8c1398b0c2c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240507, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504) 2024-05-09T04:00:11.430 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:11 smithi078 podman[61718]: 2024-05-09 04:00:11.112053275 +0000 UTC m=+0.381415559 container start cc8cc8e980a83cf6112186e033829bf618c9e2bf96fd00abcb9f8c1398b0c2c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240507, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-05-09T04:00:11.430 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mgr[61747]: -- 172.21.15.78:0/2760121585 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f18e4f05a0 con 0x55f18e4c5000 2024-05-09T04:00:11.430 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:11 smithi078 bash[61718]: cc8cc8e980a83cf6112186e033829bf618c9e2bf96fd00abcb9f8c1398b0c2c8 2024-05-09T04:00:11.430 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:11 smithi078 systemd[1]: Started Ceph mgr.y for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:00:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:11 smithi132 ceph-mon[23807]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:00:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:11 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:11 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:11 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:11 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:11.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:11 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:00:11.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:11 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:00:11.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:11 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[27629]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:00:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:00:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:00:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:11.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[22905]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:00:11.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:11.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:11.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:11.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:11.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:00:11.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:00:11.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:11 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:12.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:12 smithi132 ceph-mon[23807]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:00:12.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:12 smithi132 ceph-mon[23807]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:00:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:12 smithi078 ceph-mon[22905]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:00:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:12 smithi078 ceph-mon[22905]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:00:12.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:12 smithi078 ceph-mon[27629]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:00:12.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:12 smithi078 ceph-mon[27629]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:00:13.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:13 smithi132 ceph-mon[23807]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:00:13.571 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:13 smithi078 ceph-mon[27629]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:00:13.571 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:13 smithi078 ceph-mon[22905]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:00:13.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:00:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:00:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:00:14.914 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T04:00:14.537+0000 7f4ffcefb700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:00:14.914 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Creating ceph-iscsi config... 2024-05-09T04:00:14.914 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:00:14.914 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:00:14.914 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:00:14.914 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:14.914 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:14.915 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T04:00:14.915 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:00:14.915 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return _run_code(code, main_globals, None, 2024-05-09T04:00:14.915 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:00:14.915 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: exec(code, run_globals) 2024-05-09T04:00:14.915 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:00:14.915 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:00:14.915 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:00:14.915 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:00:14.915 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:00:14.915 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:00:14.915 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:00:14.916 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:14.916 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:14.916 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T04:00:14.916 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T04:00:14.916 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T04:00:14.916 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T04:00:14.916 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T04:00:14.916 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T04:00:14.916 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return self.event_loop.get_result(coro, timeout) 2024-05-09T04:00:14.916 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T04:00:14.916 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return future.result(timeout) 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return self.__get_result() 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: raise self._exception 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: use_current_daemon_image=reconfig, 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Creating ceph-iscsi config... 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:14.917 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return _run_code(code, main_globals, None, 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: exec(code, run_globals) 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:14.918 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:14.919 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:00:14] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-05-09T04:00:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:15 smithi132 ceph-mon[23807]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:15 smithi132 ceph-mon[23807]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Creating ceph-iscsi config... 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: exec(code, run_globals) 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:00:15.755 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T04:00:15.756 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: return future.result(timeout) 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: return self.__get_result() 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: raise self._exception 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: use_current_daemon_image=reconfig, 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Creating ceph-iscsi config... 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:00:15.757 INFO:journalctl@ceph.mon.b.smithi132.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: exec(code, run_globals) 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:00:15.758 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:00:15.759 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:00:15.759 INFO:journalctl@ceph.mon.b.smithi132.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:15.759 INFO:journalctl@ceph.mon.b.smithi132.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:15.759 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:15 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:00:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:15 smithi078 ceph-mon[22905]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:00:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:15 smithi078 ceph-mon[22905]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:00:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T04:00:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:00:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:00:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:00:15.762 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: exec(code, run_globals) 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:15.763 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: return future.result(timeout) 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: return self.__get_result() 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: raise self._exception 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: use_current_daemon_image=reconfig, 2024-05-09T04:00:15.764 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: exec(code, run_globals) 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:00:15.765 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:00:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:00:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:00:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:00:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:00:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:00:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:15.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:15 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:00:15.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:15 smithi078 ceph-mon[27629]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:00:15.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:15 smithi078 ceph-mon[27629]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:00:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T04:00:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:00:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:00:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:00:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T04:00:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:00:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:00:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:00:15.767 INFO:journalctl@ceph.mon.c.smithi078.stdout: exec(code, run_globals) 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T04:00:15.768 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: return future.result(timeout) 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: return self.__get_result() 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: raise self._exception 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: use_current_daemon_image=reconfig, 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:00:15.769 INFO:journalctl@ceph.mon.c.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: exec(code, run_globals) 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:00:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:00:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:00:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:00:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:15 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:00:17.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:17 smithi132 ceph-mon[23807]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:00:17.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:17 smithi078 ceph-mon[27629]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:00:17.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:17 smithi078 ceph-mon[22905]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:00:19.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:00:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:00:18.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:00:19.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:18 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:00:18] ENGINE Bus STARTING 2024-05-09T04:00:19.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:18 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: CherryPy Checker: 2024-05-09T04:00:19.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:18 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: The Application mounted at '' has an empty config. 2024-05-09T04:00:19.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:18 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:00:19.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:18 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:00:18] ENGINE Serving on http://:::9283 2024-05-09T04:00:19.012 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:00:18 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:00:18] ENGINE Bus STARTED 2024-05-09T04:00:19.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:00:19.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:19.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:19.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:19.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:00:19.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 71 op/s 2024-05-09T04:00:19.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:19.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: Standby manager daemon y restarted 2024-05-09T04:00:19.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: Standby manager daemon y started 2024-05-09T04:00:19.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-09T04:00:19.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:00:19.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-09T04:00:19.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:00:19.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:00:19.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:00:19.605 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:19.605 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[27629]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi078.vnplzc on smithi078 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-05-09T04:00:19.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:00:19.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:19.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:19.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:19.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:00:19.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 71 op/s 2024-05-09T04:00:19.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:19.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: Standby manager daemon y restarted 2024-05-09T04:00:19.605 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: Standby manager daemon y started 2024-05-09T04:00:19.606 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-09T04:00:19.606 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:00:19.606 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-09T04:00:19.606 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:00:19.606 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:00:19.606 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:00:19.606 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:19.606 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:19 smithi078 ceph-mon[22905]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi078.vnplzc on smithi078 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-05-09T04:00:20.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 71 op/s 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: Standby manager daemon y restarted 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: Standby manager daemon y started 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:00:20.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:00:20.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:00:20.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:20.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:19 smithi132 ceph-mon[23807]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi078.vnplzc on smithi078 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-05-09T04:00:20.752 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:20 smithi078 ceph-mon[22905]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:00:20.752 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:20 smithi078 ceph-mon[22905]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:00:20.752 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:20 smithi078 ceph-mon[22905]: mgrmap e27: x(active, since 3m), standbys: y 2024-05-09T04:00:20.752 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:20 smithi078 ceph-mon[27629]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:00:20.752 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:20 smithi078 ceph-mon[27629]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:00:20.752 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:20 smithi078 ceph-mon[27629]: mgrmap e27: x(active, since 3m), standbys: y 2024-05-09T04:00:21.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:20 smithi132 ceph-mon[23807]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:00:21.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:20 smithi132 ceph-mon[23807]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:00:21.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:20 smithi132 ceph-mon[23807]: mgrmap e27: x(active, since 3m), standbys: y 2024-05-09T04:00:21.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:21 smithi078 ceph-mon[22905]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 71 op/s 2024-05-09T04:00:21.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:21 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:21.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:21 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:21.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:21 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:00:21.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:21 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:00:21.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:21 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:00:21.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:21 smithi078 ceph-mon[27629]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 71 op/s 2024-05-09T04:00:21.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:21 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:21.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:21 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:21.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:21 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:00:21.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:21 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:00:21.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:21 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:00:22.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:21 smithi132 ceph-mon[23807]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 71 op/s 2024-05-09T04:00:22.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:21 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:22.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:21 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:22.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:21 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:00:22.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:21 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:00:22.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:21 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:00:22.861 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:22 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:00:22.862 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:22 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:00:22.862 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:22 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3735608799' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:00:22.862 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:22 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:22.862 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:22 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:00:22.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:22 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:00:22.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:22 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:00:22.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:22 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3735608799' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:00:22.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:22 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:22.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:22 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:00:23.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:22 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:00:23.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:22 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:00:23.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:22 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3735608799' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:00:23.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:22 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:23.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:22 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:00:23.946 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:23 smithi078 ceph-mon[22905]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 71 op/s 2024-05-09T04:00:23.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:23 smithi078 ceph-mon[27629]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 71 op/s 2024-05-09T04:00:24.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:00:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:00:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:00:24.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:23 smithi132 ceph-mon[23807]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 0 B/s wr, 71 op/s 2024-05-09T04:00:25.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:00:24] "GET /metrics HTTP/1.1" 200 38053 "" "Prometheus/2.33.4" 2024-05-09T04:00:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:25 smithi078 ceph-mon[22905]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-05-09T04:00:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:25 smithi078 ceph-mon[27629]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-05-09T04:00:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:25 smithi132 ceph-mon[23807]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-05-09T04:00:27.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:26 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:27.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:26 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:27.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:26 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:27.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:26 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:00:27.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:26 smithi132 ceph-mon[23807]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-05-09T04:00:27.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:26 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:27.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:26 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:27.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:26 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:27.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:26 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:27.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:26 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:00:27.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:26 smithi078 ceph-mon[22905]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-05-09T04:00:27.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:26 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:26 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:26 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:26 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:00:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:26 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:00:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:26 smithi078 ceph-mon[27629]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-05-09T04:00:27.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:26 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:27 smithi078 ceph-mon[22905]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi078.vnplzc on smithi078 is in unknown state']) 2024-05-09T04:00:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:27 smithi078 ceph-mon[22905]: Cluster is now healthy 2024-05-09T04:00:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:27 smithi078 ceph-mon[27629]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi078.vnplzc on smithi078 is in unknown state']) 2024-05-09T04:00:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:27 smithi078 ceph-mon[27629]: Cluster is now healthy 2024-05-09T04:00:28.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:27 smithi132 ceph-mon[23807]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi078.vnplzc on smithi078 is in unknown state']) 2024-05-09T04:00:28.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:27 smithi132 ceph-mon[23807]: Cluster is now healthy 2024-05-09T04:00:29.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:00:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:00:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:00:29.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:29 smithi132 ceph-mon[23807]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-05-09T04:00:29.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:29 smithi078 ceph-mon[22905]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-05-09T04:00:29.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:29 smithi078 ceph-mon[27629]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-05-09T04:00:31.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:31 smithi132 ceph-mon[23807]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-05-09T04:00:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:31 smithi078 ceph-mon[22905]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-05-09T04:00:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:31 smithi078 ceph-mon[27629]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-05-09T04:00:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:32 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:00:32.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:32 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:00:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:32 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:00:33.406 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:33 smithi132 ceph-mon[23807]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-05-09T04:00:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:33 smithi078 ceph-mon[22905]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-05-09T04:00:33.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:33 smithi078 ceph-mon[27629]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-05-09T04:00:33.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:00:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:00:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:00:35.086 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:00:34] "GET /metrics HTTP/1.1" 200 38053 "" "Prometheus/2.33.4" 2024-05-09T04:00:35.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:35 smithi132 ceph-mon[23807]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-09T04:00:35.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:35 smithi078 ceph-mon[22905]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-09T04:00:35.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:35 smithi078 ceph-mon[27629]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-09T04:00:37.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:37 smithi132 ceph-mon[23807]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 912 B/s rd, 0 op/s 2024-05-09T04:00:37.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:37 smithi078 ceph-mon[22905]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 912 B/s rd, 0 op/s 2024-05-09T04:00:37.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:37 smithi078 ceph-mon[27629]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 912 B/s rd, 0 op/s 2024-05-09T04:00:38.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:00:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:00:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:00:38.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:38 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:38.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:38 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:00:38.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:38 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:38.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:38 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:00:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:38 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:00:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:38 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:00:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:39 smithi132 ceph-mon[23807]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:39.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:39 smithi078 ceph-mon[22905]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:39.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:39 smithi078 ceph-mon[27629]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:41.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:41 smithi132 ceph-mon[23807]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:00:41.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:41 smithi078 ceph-mon[22905]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:00:41.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:41 smithi078 ceph-mon[27629]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:00:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:43 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:00:43.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:43 smithi132 ceph-mon[23807]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:43.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:00:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:00:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:00:43.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:43 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:00:43.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:43 smithi078 ceph-mon[22905]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:43.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:43 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:00:43.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:43 smithi078 ceph-mon[27629]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:45.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:44 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:00:44] "GET /metrics HTTP/1.1" 200 38057 "" "Prometheus/2.33.4" 2024-05-09T04:00:45.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:45 smithi132 ceph-mon[23807]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:00:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:45 smithi078 ceph-mon[22905]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:00:45.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:45 smithi078 ceph-mon[27629]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:00:47.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:47 smithi132 ceph-mon[23807]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:47.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:47 smithi078 ceph-mon[22905]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:47.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:47 smithi078 ceph-mon[27629]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:49.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:00:48 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:00:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:00:49.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:49 smithi078 ceph-mon[22905]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:49.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:49 smithi078 ceph-mon[27629]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:50.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:49 smithi132 ceph-mon[23807]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:51.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:51 smithi078 ceph-mon[22905]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:00:51.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:51 smithi078 ceph-mon[27629]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:00:52.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:51 smithi132 ceph-mon[23807]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:00:53.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:52 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:00:53.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:52 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:00:53.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:52 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:00:54.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:53 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:00:54.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:53 smithi132 ceph-mon[23807]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:54.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:00:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:00:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:00:54.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:53 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:00:54.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:53 smithi078 ceph-mon[22905]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:54.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:53 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:00:54.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:53 smithi078 ceph-mon[27629]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:55.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:00:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:00:54] "GET /metrics HTTP/1.1" 200 38056 "" "Prometheus/2.33.4" 2024-05-09T04:00:56.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:55 smithi132 ceph-mon[23807]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:00:56.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:55 smithi078 ceph-mon[22905]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:00:56.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:55 smithi078 ceph-mon[27629]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:00:58.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:57 smithi132 ceph-mon[23807]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:58.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:57 smithi078 ceph-mon[22905]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:58.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:57 smithi078 ceph-mon[27629]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:00:59.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:00:58 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:00:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:00:59 smithi132 ceph-mon[23807]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:00.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:00:59 smithi078 ceph-mon[22905]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:00.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:00:59 smithi078 ceph-mon[27629]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:01.798 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:01 smithi132 ceph-mon[23807]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:01.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:01 smithi078 ceph-mon[27629]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:01.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:01 smithi078 ceph-mon[22905]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:04.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:01:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:01:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:04.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:03 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:04.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:03 smithi132 ceph-mon[23807]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:04.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:03 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:04.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:03 smithi078 ceph-mon[27629]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:04.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:03 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:04.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:03 smithi078 ceph-mon[22905]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:05.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:01:04 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:01:04] "GET /metrics HTTP/1.1" 200 38056 "" "Prometheus/2.33.4" 2024-05-09T04:01:06.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:05 smithi132 ceph-mon[23807]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:06.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:05 smithi078 ceph-mon[22905]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:06.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:05 smithi078 ceph-mon[27629]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:08.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:07 smithi132 ceph-mon[23807]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:08.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:07 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:01:08.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:07 smithi078 ceph-mon[22905]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:08.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:07 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:01:08.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:07 smithi078 ceph-mon[27629]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:08.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:07 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:01:09.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:01:08 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:01:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:09.813 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:09 smithi078 ceph-mon[22905]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:09.814 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:09 smithi078 ceph-mon[27629]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:10.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:09 smithi132 ceph-mon[23807]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:12.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:11 smithi132 ceph-mon[23807]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:12.050 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:11 smithi078 ceph-mon[27629]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:12.051 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:11 smithi078 ceph-mon[22905]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:14.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:01:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:01:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:14.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:13 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:14.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:13 smithi132 ceph-mon[23807]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:14.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:13 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:14.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:13 smithi078 ceph-mon[22905]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:14.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:13 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:14.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:13 smithi078 ceph-mon[27629]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:15.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:01:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:01:14] "GET /metrics HTTP/1.1" 200 38056 "" "Prometheus/2.33.4" 2024-05-09T04:01:16.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:15 smithi132 ceph-mon[23807]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:16.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:15 smithi078 ceph-mon[22905]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:16.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:15 smithi078 ceph-mon[27629]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:18.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:17 smithi132 ceph-mon[23807]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:18.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:17 smithi078 ceph-mon[22905]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:18.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:17 smithi078 ceph-mon[27629]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:19.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:01:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:01:18.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:20.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:19 smithi132 ceph-mon[23807]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:20.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:19 smithi078 ceph-mon[22905]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:20.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:19 smithi078 ceph-mon[27629]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:22.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:21 smithi132 ceph-mon[23807]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:22.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:21 smithi078 ceph-mon[27629]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:22.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:21 smithi078 ceph-mon[22905]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:23.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:22 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:01:23.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:22 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:01:23.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:22 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:01:24.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:23 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:24.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:23 smithi132 ceph-mon[23807]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:24.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:01:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:01:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:24.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:23 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:24.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:23 smithi078 ceph-mon[27629]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:24.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:23 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:24.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:23 smithi078 ceph-mon[22905]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:25.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:01:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:01:24] "GET /metrics HTTP/1.1" 200 38057 "" "Prometheus/2.33.4" 2024-05-09T04:01:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:25 smithi132 ceph-mon[23807]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:26.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:25 smithi078 ceph-mon[27629]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:26.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:25 smithi078 ceph-mon[22905]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:27.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:26 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:01:27.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:26 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:01:27.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:26 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:01:27.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:26 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:01:27.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:26 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:01:27.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:26 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:01:27.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:26 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:01:27.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:26 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:01:27.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:26 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:01:27.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:26 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:01:27.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:26 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:01:27.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:26 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:01:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:27 smithi132 ceph-mon[23807]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:27 smithi078 ceph-mon[22905]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:27 smithi078 ceph-mon[27629]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:29.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:01:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:01:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:30.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:29 smithi132 ceph-mon[23807]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:30.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:29 smithi078 ceph-mon[22905]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:30.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:29 smithi078 ceph-mon[27629]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:32.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:31 smithi132 ceph-mon[23807]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:32.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:31 smithi078 ceph-mon[22905]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:32.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:31 smithi078 ceph-mon[27629]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:34.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:01:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:01:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:34.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:33 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:34.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:33 smithi132 ceph-mon[23807]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:34.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:33 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:34.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:33 smithi078 ceph-mon[22905]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:33 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:33 smithi078 ceph-mon[27629]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:35.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:01:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:01:34] "GET /metrics HTTP/1.1" 200 38057 "" "Prometheus/2.33.4" 2024-05-09T04:01:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:35 smithi132 ceph-mon[23807]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:35 smithi078 ceph-mon[22905]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:35 smithi078 ceph-mon[27629]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:37 smithi132 ceph-mon[23807]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:37 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:01:38.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:37 smithi078 ceph-mon[22905]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:38.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:37 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:01:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:37 smithi078 ceph-mon[27629]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:37 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:01:39.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:01:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:01:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:40.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:39 smithi132 ceph-mon[23807]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:40.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:39 smithi078 ceph-mon[27629]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:40.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:39 smithi078 ceph-mon[22905]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:42.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:41 smithi132 ceph-mon[23807]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:42.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:41 smithi078 ceph-mon[22905]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:42.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:41 smithi078 ceph-mon[27629]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:44.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:43 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:44.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:43 smithi132 ceph-mon[23807]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:44.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:01:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:01:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:44.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:43 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:44.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:43 smithi078 ceph-mon[22905]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:44.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:43 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:44.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:43 smithi078 ceph-mon[27629]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:44.912 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:44 smithi132 ceph-mon[23807]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:45.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:44 smithi078 ceph-mon[22905]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:45.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:44 smithi078 ceph-mon[27629]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:45.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:01:44 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:01:44] "GET /metrics HTTP/1.1" 200 38052 "" "Prometheus/2.33.4" 2024-05-09T04:01:47.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:46 smithi132 ceph-mon[23807]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:47.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:46 smithi078 ceph-mon[27629]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:47.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:46 smithi078 ceph-mon[22905]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:49.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:01:48 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:01:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:49.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:49 smithi132 ceph-mon[23807]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:49.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:49 smithi078 ceph-mon[27629]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:49.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:49 smithi078 ceph-mon[22905]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:51 smithi132 ceph-mon[23807]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:51 smithi078 ceph-mon[22905]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:51.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:51 smithi078 ceph-mon[27629]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:53 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:53 smithi132 ceph-mon[23807]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:01:53.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:53 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:53.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:53 smithi078 ceph-mon[22905]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:53.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:01:53.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:53 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:01:53.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:53 smithi078 ceph-mon[27629]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:53.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:01:54.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:01:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:01:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:55.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:55 smithi132 ceph-mon[23807]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:55.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:01:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:01:54] "GET /metrics HTTP/1.1" 200 38056 "" "Prometheus/2.33.4" 2024-05-09T04:01:55.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:55 smithi078 ceph-mon[22905]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:55.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:55 smithi078 ceph-mon[27629]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:01:57.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:57 smithi132 ceph-mon[23807]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:57.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:57 smithi078 ceph-mon[22905]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:57.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:57 smithi078 ceph-mon[27629]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:59.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:01:58 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:01:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:01:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:01:59 smithi132 ceph-mon[23807]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:01:59 smithi078 ceph-mon[22905]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:01:59.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:01:59 smithi078 ceph-mon[27629]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:01.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:01 smithi132 ceph-mon[23807]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:01.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:01 smithi078 ceph-mon[22905]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:01.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:01 smithi078 ceph-mon[27629]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:03 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:03 smithi132 ceph-mon[23807]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:03.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:03 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:03.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:03 smithi078 ceph-mon[27629]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:03.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:03 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:03.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:03 smithi078 ceph-mon[22905]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:04.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:02:03 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:02:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi078\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", domainname=\"(none)\", instance=\"smithi078\", job=\"node\", machine=\"x86_64\", nodename=\"smithi078\", release=\"5.14.0-444.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:02:05.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:05 smithi132 ceph-mon[23807]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:05.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:02:04 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:02:04] "GET /metrics HTTP/1.1" 200 38056 "" "Prometheus/2.33.4" 2024-05-09T04:02:05.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:05 smithi078 ceph-mon[22905]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:05.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:05 smithi078 ceph-mon[27629]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:07.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:07 smithi132 ceph-mon[23807]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:07.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:07 smithi078 ceph-mon[22905]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:07.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:07 smithi078 ceph-mon[27629]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:08.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:08 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:02:08.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:08 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:02:08.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:08 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:02:09.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:02:08 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:02:08.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"172.21.15.132:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"2777c79e-0db6-11ef-bc97-c7b262605968\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-09T04:02:09.414 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:09 smithi078 ceph-mon[27629]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:09.414 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:09 smithi078 ceph-mon[22905]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:09.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:09 smithi132 ceph-mon[23807]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:11.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:11 smithi132 ceph-mon[23807]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:11.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:11 smithi078 ceph-mon[27629]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:11.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:11 smithi078 ceph-mon[22905]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:13.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:13 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:13.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:13 smithi132 ceph-mon[23807]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:13.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:13 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:13.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:13 smithi078 ceph-mon[27629]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:13.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:13 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:13.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:13 smithi078 ceph-mon[22905]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:15.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:15 smithi132 ceph-mon[23807]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:15.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:02:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:02:14] "GET /metrics HTTP/1.1" 200 38054 "" "Prometheus/2.33.4" 2024-05-09T04:02:15.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:15 smithi078 ceph-mon[22905]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:15 smithi078 ceph-mon[27629]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:17.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:17 smithi132 ceph-mon[23807]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:17.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:17 smithi078 ceph-mon[22905]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:17.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:17 smithi078 ceph-mon[27629]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:19.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:19 smithi132 ceph-mon[23807]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:19.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:19 smithi078 ceph-mon[22905]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:19.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:19 smithi078 ceph-mon[27629]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:21.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:21 smithi132 ceph-mon[23807]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:21 smithi078 ceph-mon[22905]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:21.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:21 smithi078 ceph-mon[27629]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:23.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:23 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:23.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:23 smithi132 ceph-mon[23807]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:23.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:23 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:02:23.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:23 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:23.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:23 smithi078 ceph-mon[22905]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:23.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:23 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:02:23.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:23 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:23.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:23 smithi078 ceph-mon[27629]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:23.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:23 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:02:25.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:25 smithi132 ceph-mon[23807]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:25.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:02:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:02:24] "GET /metrics HTTP/1.1" 200 38053 "" "Prometheus/2.33.4" 2024-05-09T04:02:25.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:25 smithi078 ceph-mon[27629]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:25 smithi078 ceph-mon[22905]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:27.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:27 smithi132 ceph-mon[23807]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:27.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:27 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:02:27.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:27 smithi078 ceph-mon[22905]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:27.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:27 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:02:27.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:27 smithi078 ceph-mon[27629]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:27.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:27 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:02:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:28 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:02:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:28 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:02:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:28 smithi132 ceph-mon[23807]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:02:28.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:28 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:02:28.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:28 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:02:28.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:28 smithi078 ceph-mon[22905]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:02:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:28 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:02:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:28 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:02:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:28 smithi078 ceph-mon[27629]: from='mgr.24740 ' entity='mgr.x' 2024-05-09T04:02:29.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:29 smithi132 ceph-mon[23807]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:29.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:29 smithi078 ceph-mon[22905]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:29.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:29 smithi078 ceph-mon[27629]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:31.704 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:31 smithi078 ceph-mon[22905]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:31.704 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:31 smithi078 ceph-mon[27629]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:31 smithi132 ceph-mon[23807]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:33.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:33 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:33.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:33 smithi132 ceph-mon[23807]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:33.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:33 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:33.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:33 smithi078 ceph-mon[27629]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:33.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:33 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:33.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:33 smithi078 ceph-mon[22905]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:35.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:02:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:02:34] "GET /metrics HTTP/1.1" 200 38053 "" "Prometheus/2.33.4" 2024-05-09T04:02:35.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:35 smithi132 ceph-mon[23807]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:35.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:35 smithi078 ceph-mon[22905]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:35.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:35 smithi078 ceph-mon[27629]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:37 smithi132 ceph-mon[23807]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:37.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:37 smithi078 ceph-mon[27629]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:37 smithi078 ceph-mon[22905]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:38.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:38 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:02:38.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:38 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:02:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:38 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:02:39.663 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:39 smithi078 ceph-mon[22905]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:39.664 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:39 smithi078 ceph-mon[27629]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:39 smithi132 ceph-mon[23807]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:41.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:41 smithi078 ceph-mon[22905]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:41.709 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:41 smithi078 ceph-mon[27629]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:41.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:41 smithi132 ceph-mon[23807]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:43.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:43 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:43.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:43 smithi132 ceph-mon[23807]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:43.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:43 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:43.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:43 smithi078 ceph-mon[22905]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:43.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:43 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:43.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:43 smithi078 ceph-mon[27629]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:45.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:02:44 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:02:44] "GET /metrics HTTP/1.1" 200 38058 "" "Prometheus/2.33.4" 2024-05-09T04:02:45.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:45 smithi132 ceph-mon[23807]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:45.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:45 smithi078 ceph-mon[22905]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:45.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:45 smithi078 ceph-mon[27629]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:47.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:47 smithi132 ceph-mon[23807]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:47.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:47 smithi078 ceph-mon[22905]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:47.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:47 smithi078 ceph-mon[27629]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:49.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:49 smithi132 ceph-mon[23807]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:49.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:49 smithi078 ceph-mon[22905]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:49.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:49 smithi078 ceph-mon[27629]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:51.710 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:51 smithi078 ceph-mon[22905]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:51.711 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:51 smithi078 ceph-mon[27629]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:51.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:51 smithi132 ceph-mon[23807]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:53.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:53 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:53.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:53 smithi132 ceph-mon[23807]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:53.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:53 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:02:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:53 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:53 smithi078 ceph-mon[27629]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:53 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:02:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:53 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:02:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:53 smithi078 ceph-mon[22905]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:53 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:02:55.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:02:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:02:54] "GET /metrics HTTP/1.1" 200 38042 "" "Prometheus/2.33.4" 2024-05-09T04:02:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:55 smithi132 ceph-mon[23807]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:55 smithi078 ceph-mon[22905]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:55.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:55 smithi078 ceph-mon[27629]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:02:57.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:57 smithi132 ceph-mon[23807]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:57.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:57 smithi078 ceph-mon[22905]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:57.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:57 smithi078 ceph-mon[27629]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:59.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:02:59 smithi132 ceph-mon[23807]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:59.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:02:59 smithi078 ceph-mon[22905]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:02:59.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:02:59 smithi078 ceph-mon[27629]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:01.721 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:01 smithi078 ceph-mon[22905]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:01.722 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:01 smithi078 ceph-mon[27629]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:01 smithi132 ceph-mon[23807]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:03.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:03 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:03.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:03 smithi132 ceph-mon[23807]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:03.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:03 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:03.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:03 smithi078 ceph-mon[22905]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:03.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:03 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:03.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:03 smithi078 ceph-mon[27629]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:05.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:04 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:03:04] "GET /metrics HTTP/1.1" 200 38042 "" "Prometheus/2.33.4" 2024-05-09T04:03:05.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:05 smithi078 ceph-mon[22905]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:05.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:05 smithi078 ceph-mon[27629]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:05 smithi132 ceph-mon[23807]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:05.940 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T04:03:07.596 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:07 smithi078 ceph-mon[22905]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:07.596 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:07 smithi078 ceph-mon[27629]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:07.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:07 smithi132 ceph-mon[23807]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:07.924 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (5m) 2m ago 13m 25.5M - 0.23.0 ba2b418f427c f44cb50f5f35 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (5m) 3m ago 13m 50.2M - 8.3.5 dad864ee21e9 33b2a450dbe1 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (2m) 2m ago 12m 48.3M - 3.5 e1d6a67b021e f0c32e17125f 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283 running (8m) 3m ago 18m 522M - 19.0.0-3631-gc66c553a af4a4df59e0a 4a3f6038cf25 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (2m) 2m ago 20m 462M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (20m) 2m ago 20m 60.9M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (18m) 3m ago 18m 50.9M 2048M 17.2.0 e1d6a67b021e a71bae52559d 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (18m) 2m ago 18m 54.2M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (6m) 2m ago 14m 18.1M - 1.3.1 1dbe0e931976 4a3cc1414851 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (5m) 3m ago 14m 17.3M - 1.3.1 1dbe0e931976 666630a53114 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (18m) 2m ago 18m 53.0M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (17m) 2m ago 17m 56.4M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (17m) 2m ago 17m 49.9M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:03:07.925 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (16m) 2m ago 16m 52.8M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:03:07.926 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (16m) 3m ago 16m 53.4M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:03:07.926 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (16m) 3m ago 16m 50.8M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:03:07.926 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (15m) 3m ago 15m 50.6M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:03:07.926 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (15m) 3m ago 15m 54.0M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:03:07.926 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (5m) 3m ago 14m 71.1M - 2.33.4 514e6a882f6e 860656079d4c 2024-05-09T04:03:07.926 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (13m) 2m ago 13m 74.8M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:03:07.926 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (12m) 3m ago 12m 75.4M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:03:08.555 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:08 smithi078 ceph-mon[22905]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:03:08.556 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:08 smithi078 ceph-mon[27629]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:03:08.557 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions' 2024-05-09T04:03:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:08 smithi132 ceph-mon[23807]: from='mgr.24740 172.21.15.132:0/1882067085' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:03:09.742 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:09 smithi078 ceph-mon[22905]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:09.784 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:09 smithi078 ceph-mon[22905]: from='client.15267 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:03:09.784 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:09 smithi078 ceph-mon[27629]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:09.784 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:09 smithi078 ceph-mon[27629]: from='client.15267 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:03:09.785 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:09 smithi132 ceph-mon[23807]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:09.785 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:09 smithi132 ceph-mon[23807]: from='client.15267 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:03:10.624 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:03:10.624 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": {}, 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:03:10.625 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-09T04:03:10.626 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:03:10.626 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:03:10.626 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:03:11.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph health detail' 2024-05-09T04:03:11.733 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:11 smithi078 ceph-mon[22905]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:11.733 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:11 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/531485800' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:03:11.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:11 smithi078 ceph-mon[27629]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:11.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:11 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/531485800' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:03:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:11 smithi132 ceph-mon[23807]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:11 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/531485800' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:03:13.479 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-05-09T04:03:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:13 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:13 smithi132 ceph-mon[23807]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:13.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:13 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:13.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:13 smithi078 ceph-mon[22905]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:13 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:13 smithi078 ceph-mon[27629]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:14.121 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph -s' 2024-05-09T04:03:14.660 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:14 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/631612562' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:03:14.660 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:14 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/631612562' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:03:14.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:14 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/631612562' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:03:15.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:03:14] "GET /metrics HTTP/1.1" 200 38051 "" "Prometheus/2.33.4" 2024-05-09T04:03:15.633 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:15 smithi078 ceph-mon[27629]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:15.634 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:15 smithi078 ceph-mon[22905]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:15 smithi132 ceph-mon[23807]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:16.155 INFO:teuthology.orchestra.run.smithi078.stdout: cluster: 2024-05-09T04:03:16.155 INFO:teuthology.orchestra.run.smithi078.stdout: id: 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:03:16.155 INFO:teuthology.orchestra.run.smithi078.stdout: health: HEALTH_OK 2024-05-09T04:03:16.155 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T04:03:16.155 INFO:teuthology.orchestra.run.smithi078.stdout: services: 2024-05-09T04:03:16.155 INFO:teuthology.orchestra.run.smithi078.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-05-09T04:03:16.155 INFO:teuthology.orchestra.run.smithi078.stdout: mgr: x(active, since 6m), standbys: y 2024-05-09T04:03:16.155 INFO:teuthology.orchestra.run.smithi078.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-05-09T04:03:16.156 INFO:teuthology.orchestra.run.smithi078.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-09T04:03:16.156 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T04:03:16.156 INFO:teuthology.orchestra.run.smithi078.stdout: data: 2024-05-09T04:03:16.156 INFO:teuthology.orchestra.run.smithi078.stdout: pools: 6 pools, 161 pgs 2024-05-09T04:03:16.156 INFO:teuthology.orchestra.run.smithi078.stdout: objects: 241 objects, 585 KiB 2024-05-09T04:03:16.156 INFO:teuthology.orchestra.run.smithi078.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:16.156 INFO:teuthology.orchestra.run.smithi078.stdout: pgs: 161 active+clean 2024-05-09T04:03:16.156 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T04:03:16.156 INFO:teuthology.orchestra.run.smithi078.stdout: io: 2024-05-09T04:03:16.156 INFO:teuthology.orchestra.run.smithi078.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-05-09T04:03:16.156 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T04:03:16.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:16 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/4122210235' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T04:03:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:16 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/4122210235' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T04:03:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:16 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/4122210235' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T04:03:16.831 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph mgr fail' 2024-05-09T04:03:17.390 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:17 smithi078 ceph-mon[22905]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:17.390 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:17 smithi078 ceph-mon[27629]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:17.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:17 smithi132 ceph-mon[23807]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:19.742 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:19 smithi078 ceph-mon[27629]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:19.742 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:19 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/763646934' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T04:03:19.742 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:19 smithi078 ceph-mon[27629]: osdmap e104: 8 total, 8 up, 8 in 2024-05-09T04:03:19.743 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:03:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:03:19] ENGINE Bus STOPPING 2024-05-09T04:03:19.743 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:03:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:03:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-09T04:03:19.743 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:03:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:03:19] ENGINE Bus STOPPED 2024-05-09T04:03:19.743 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:03:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:03:19] ENGINE Bus STARTING 2024-05-09T04:03:19.743 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:03:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:03:19] ENGINE Serving on http://:::9283 2024-05-09T04:03:19.743 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:03:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:03:19] ENGINE Bus STARTED 2024-05-09T04:03:19.743 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:19 smithi078 ceph-mon[22905]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:19.743 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:19 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/763646934' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T04:03:19.744 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:19 smithi078 ceph-mon[22905]: osdmap e104: 8 total, 8 up, 8 in 2024-05-09T04:03:19.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:19 smithi132 ceph-mon[23807]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:19.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:19 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/763646934' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T04:03:19.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:19 smithi132 ceph-mon[23807]: osdmap e104: 8 total, 8 up, 8 in 2024-05-09T04:03:19.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T04:03:19.400+0000 7f506ba5d700 -1 mgr handle_mgr_map I was active but no longer am 2024-05-09T04:03:19.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ignoring --setuser ceph since I am not root 2024-05-09T04:03:19.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ignoring --setgroup ceph since I am not root 2024-05-09T04:03:19.755 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:19 smithi132 ceph-mgr[44012]: -- 172.21.15.132:0/3583227447 <== mon.2 v2:172.21.15.132:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5619acb525a0 con 0x5619acb27000 2024-05-09T04:03:19.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:19.448Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.132:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.132:8765: connect: connection refused" 2024-05-09T04:03:19.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:19.448Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.132:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.132:8765: connect: connection refused" 2024-05-09T04:03:19.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:19.449Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.132:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.132:8765: connect: connection refused" 2024-05-09T04:03:19.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:19.450Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.132:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.132:8765: connect: connection refused" 2024-05-09T04:03:19.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:19.450Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.132:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.132:8765: connect: connection refused" 2024-05-09T04:03:20.125 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'sleep 180' 2024-05-09T04:03:20.651 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/763646934' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-09T04:03:20.651 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: mgrmap e28: y(active, starting, since 0.519129s) 2024-05-09T04:03:20.651 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T04:03:20.652 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: Manager daemon y is now available 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: [09/May/2024:04:03:19] ENGINE Bus STARTING 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: [09/May/2024:04:03:19] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: [09/May/2024:04:03:20] ENGINE Serving on http://172.21.15.78:8765 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[27629]: [09/May/2024:04:03:20] ENGINE Bus STARTED 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/763646934' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: mgrmap e28: y(active, starting, since 0.519129s) 2024-05-09T04:03:20.653 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T04:03:20.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T04:03:20.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: Manager daemon y is now available 2024-05-09T04:03:20.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:03:20.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:03:20.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:03:20.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T04:03:20.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: [09/May/2024:04:03:19] ENGINE Bus STARTING 2024-05-09T04:03:20.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: [09/May/2024:04:03:19] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T04:03:20.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: [09/May/2024:04:03:20] ENGINE Serving on http://172.21.15.78:8765 2024-05-09T04:03:20.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:20 smithi078 ceph-mon[22905]: [09/May/2024:04:03:20] ENGINE Bus STARTED 2024-05-09T04:03:20.676 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/763646934' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-09T04:03:20.676 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: mgrmap e28: y(active, starting, since 0.519129s) 2024-05-09T04:03:20.676 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:03:20.676 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:03:20.676 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:03:20.676 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T04:03:20.676 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:03:20.676 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:03:20.676 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:03:20.676 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:03:20.676 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:03:20.676 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: Manager daemon y is now available 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: [09/May/2024:04:03:19] ENGINE Bus STARTING 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: [09/May/2024:04:03:19] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: [09/May/2024:04:03:20] ENGINE Serving on http://172.21.15.78:8765 2024-05-09T04:03:20.677 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:20 smithi132 ceph-mon[23807]: [09/May/2024:04:03:20] ENGINE Bus STARTED 2024-05-09T04:03:21.499 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:21 smithi132 ceph-mon[23807]: mgrmap e29: y(active, since 1.52593s) 2024-05-09T04:03:21.499 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:21 smithi132 ceph-mon[23807]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:21.651 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:21 smithi078 ceph-mon[22905]: mgrmap e29: y(active, since 1.52593s) 2024-05-09T04:03:21.651 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:21 smithi078 ceph-mon[22905]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:21.651 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:21 smithi078 ceph-mon[27629]: mgrmap e29: y(active, since 1.52593s) 2024-05-09T04:03:21.651 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:21 smithi078 ceph-mon[27629]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:22 smithi132 ceph-mon[23807]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:22 smithi132 ceph-mon[23807]: mgrmap e30: y(active, since 2s) 2024-05-09T04:03:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:22 smithi132 ceph-mon[23807]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:22.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:22 smithi078 ceph-mon[22905]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:22.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:22 smithi078 ceph-mon[22905]: mgrmap e30: y(active, since 2s) 2024-05-09T04:03:22.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:22 smithi078 ceph-mon[22905]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:22.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:22 smithi078 ceph-mon[27629]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:22.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:22 smithi078 ceph-mon[27629]: mgrmap e30: y(active, since 2s) 2024-05-09T04:03:22.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:22 smithi078 ceph-mon[27629]: from='client.15252 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:24 smithi132 ceph-mon[23807]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:24 smithi132 ceph-mon[23807]: mgrmap e31: y(active, since 4s) 2024-05-09T04:03:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:24 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:24 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:24 smithi078 ceph-mon[22905]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:24 smithi078 ceph-mon[22905]: mgrmap e31: y(active, since 4s) 2024-05-09T04:03:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:24 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:24.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:24 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:24 smithi078 ceph-mon[27629]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:24 smithi078 ceph-mon[27629]: mgrmap e31: y(active, since 4s) 2024-05-09T04:03:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:24 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:24 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:26 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:26 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:26 smithi132 ceph-mon[23807]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:26 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:26 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:26 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:26 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:26 smithi078 ceph-mon[22905]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:26 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:26 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:26 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:26 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:26 smithi078 ceph-mon[27629]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:03:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:26 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:26.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:26 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:27.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:26 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:04:03:26] ENGINE Bus STARTING 2024-05-09T04:03:27.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: CherryPy Checker: 2024-05-09T04:03:27.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: The Application mounted at '' has an empty config. 2024-05-09T04:03:27.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T04:03:27.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:04:03:27] ENGINE Serving on http://:::9283 2024-05-09T04:03:27.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:04:03:27] ENGINE Bus STARTED 2024-05-09T04:03:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:03:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:03:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T04:03:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T04:03:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:03:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:03:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: Standby manager daemon x started 2024-05-09T04:03:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T04:03:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:03:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T04:03:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:03:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:03:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:27 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:03:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:03:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:03:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T04:03:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T04:03:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:03:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:03:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: Standby manager daemon x started 2024-05-09T04:03:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T04:03:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:03:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T04:03:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:03:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:03:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:03:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:03:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:03:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T04:03:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T04:03:28.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:03:28.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:03:28.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: Standby manager daemon x started 2024-05-09T04:03:28.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T04:03:28.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:03:28.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T04:03:28.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:03:28.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:03:28.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:27 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:03:28.631 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T04:03:28.631 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:03:28.631 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:03:28.631 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: mgrmap e32: y(active, since 8s), standbys: x 2024-05-09T04:03:28.631 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T04:03:28.631 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:28.631 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:28.631 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:28.631 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:28.631 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:28.631 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:03:29.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T04:03:29.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:03:29.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:03:29.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: mgrmap e32: y(active, since 8s), standbys: x 2024-05-09T04:03:29.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T04:03:29.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:29.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:29.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:29.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:29.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:29.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:03:29.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:03:29.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:03:29.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:28 smithi132 ceph-mon[23807]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:03:29.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:03:29.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:03:29.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[22905]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:03:29.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T04:03:29.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:03:29.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:03:29.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: mgrmap e32: y(active, since 8s), standbys: x 2024-05-09T04:03:29.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T04:03:29.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:29.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:29.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:29.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:29.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:29.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:03:29.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:03:29.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:03:29.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:28 smithi078 ceph-mon[27629]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:03:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:30 smithi132 ceph-mon[23807]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T04:03:31.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:30 smithi078 ceph-mon[22905]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T04:03:31.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:30 smithi078 ceph-mon[27629]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T04:03:32.026 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:03:31 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:03:31.932+0000 7fb952e78700 -1 log_channel(cephadm) log [ERR] : Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:03:33.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:32 smithi132 ceph-mon[23807]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:03:33.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:32 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:33.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:32 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:33.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:32 smithi132 ceph-mon[23807]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T04:03:33.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:32 smithi132 ceph-mon[23807]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:03:33.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:32 smithi132 ceph-mon[23807]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T04:03:33.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:32 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3210443043' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:03:33.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[22905]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:03:33.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:33.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:33.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[22905]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T04:03:33.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[22905]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:03:33.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[22905]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T04:03:33.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3210443043' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:03:33.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[27629]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:03:33.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:33.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:33.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[27629]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T04:03:33.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[27629]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:03:33.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[27629]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T04:03:33.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:32 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3210443043' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:03:35.180 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:03:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:03:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T04:03:35.180 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:35 smithi132 ceph-mon[23807]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:03:35.180 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:35 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:03:35.484 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 systemd[1]: Stopping Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:03:35.484 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 bash[52615]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus.a" found: no such container 2024-05-09T04:03:35.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:35 smithi078 ceph-mon[27629]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:03:35.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:35 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:03:35.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:35 smithi078 ceph-mon[22905]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:03:35.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:35 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:03:35.754 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:35.483Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-09T04:03:35.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:35.483Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-09T04:03:35.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:35.483Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-09T04:03:35.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:35.483Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-09T04:03:35.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:35.483Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-09T04:03:35.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:35.483Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-09T04:03:35.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:35.483Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-09T04:03:35.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:35.483Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-09T04:03:35.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:35.483Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-09T04:03:35.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:35.485Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-09T04:03:35.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:35.485Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-09T04:03:35.755 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[47200]: ts=2024-05-09T04:03:35.485Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-09T04:03:35.756 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 podman[52627]: 2024-05-09 04:03:35.493764268 +0000 UTC m=+0.143657617 container died 860656079d4cfb4fe4ee4fbe28d82fe5500220e84181752e5d2e876b60acefdc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:03:36.126 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 podman[52627]: 2024-05-09 04:03:35.882994357 +0000 UTC m=+0.532887709 container cleanup 860656079d4cfb4fe4ee4fbe28d82fe5500220e84181752e5d2e876b60acefdc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:03:36.126 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 bash[52627]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a 2024-05-09T04:03:36.126 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:35 smithi132 bash[52656]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus.a" found: no such container 2024-05-09T04:03:36.377 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 podman[52641]: 2024-05-09 04:03:36.125733622 +0000 UTC m=+0.632246810 container remove 860656079d4cfb4fe4ee4fbe28d82fe5500220e84181752e5d2e876b60acefdc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:03:36.377 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-09T04:03:36.378 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 systemd[1]: Stopped Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:03:36.378 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service: Consumed 2.036s CPU time. 2024-05-09T04:03:36.378 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 systemd[1]: Starting Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:03:36.378 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 podman[52728]: 2024-05-09 04:03:36.276898699 +0000 UTC m=+0.020146721 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-09T04:03:36.378 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 podman[52728]: 2024-05-09 04:03:36.376859576 +0000 UTC m=+0.120107578 container create a1ad9e190e07d75e0b959051c9e644506ed3e7071924ba61f38efeb5b1c7deb0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:03:37.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 podman[52728]: 2024-05-09 04:03:36.64588097 +0000 UTC m=+0.389128973 container init a1ad9e190e07d75e0b959051c9e644506ed3e7071924ba61f38efeb5b1c7deb0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:03:37.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 podman[52728]: 2024-05-09 04:03:36.649054348 +0000 UTC m=+0.392302359 container start a1ad9e190e07d75e0b959051c9e644506ed3e7071924ba61f38efeb5b1c7deb0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:03:37.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.672Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-09T04:03:37.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.672Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-09T04:03:37.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.672Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-09T04:03:37.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.672Z caller=main.go:518 level=info host_details="(Linux 5.14.0-444.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024 x86_64 smithi132 (none))" 2024-05-09T04:03:37.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.672Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-09T04:03:37.005 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.672Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-09T04:03:37.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.674Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-09T04:03:37.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.675Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-09T04:03:37.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.675Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-09T04:03:37.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.679Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-09T04:03:37.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.680Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.241481ms 2024-05-09T04:03:37.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.680Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-09T04:03:37.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 bash[52728]: a1ad9e190e07d75e0b959051c9e644506ed3e7071924ba61f38efeb5b1c7deb0 2024-05-09T04:03:37.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 systemd[1]: Started Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:03:37.006 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:36.890Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-05-09T04:03:37.375 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:37.144Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-05-09T04:03:37.376 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:37.375Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-05-09T04:03:37.376 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:37.375Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-05-09T04:03:37.376 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:37 smithi132 ceph-mon[23807]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:03:37.376 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:37 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:37.376 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:37 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:37.376 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:37 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:03:37.376 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:37 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:37.376 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:37 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:03:37.376 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:37 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T04:03:37.376 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:37 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:37.376 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:37 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:03:37.377 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:37 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:03:37.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[22905]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:03:37.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:37.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:37.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:03:37.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:37.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:03:37.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T04:03:37.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:37.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:03:37.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:03:37.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[27629]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:03:37.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:37.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:37.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:03:37.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:37.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:03:37.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T04:03:37.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:37.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:03:37.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:37 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:03:37.659 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:37.375Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=79.332µs wal_replay_duration=694.659179ms total_replay_duration=696.010644ms 2024-05-09T04:03:37.660 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:37.380Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-09T04:03:37.660 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:37.380Z caller=main.go:947 level=info msg="TSDB started" 2024-05-09T04:03:37.660 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:37.380Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-09T04:03:37.660 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:37.405Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.086148ms db_storage=812ns remote_storage=1.719µs web_handler=400ns query_engine=620ns scrape=395.978µs scrape_sd=96.249µs notify=11.57µs notify_sd=8.491µs rules=24.19111ms 2024-05-09T04:03:37.660 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:03:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:03:37.405Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-09T04:03:38.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:38 smithi078 ceph-mon[22905]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:03:38.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:38 smithi078 ceph-mon[22905]: Adding iSCSI gateway http://:@172.21.15.78:5000 to Dashboard 2024-05-09T04:03:38.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:38 smithi078 ceph-mon[22905]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:03:38.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:38 smithi078 ceph-mon[22905]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T04:03:38.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:38 smithi078 ceph-mon[22905]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:03:38.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:38 smithi078 ceph-mon[27629]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:03:38.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:38 smithi078 ceph-mon[27629]: Adding iSCSI gateway http://:@172.21.15.78:5000 to Dashboard 2024-05-09T04:03:38.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:38 smithi078 ceph-mon[27629]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:03:38.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:38 smithi078 ceph-mon[27629]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T04:03:38.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:38 smithi078 ceph-mon[27629]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:03:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:38 smithi132 ceph-mon[23807]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:03:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:38 smithi132 ceph-mon[23807]: Adding iSCSI gateway http://:@172.21.15.78:5000 to Dashboard 2024-05-09T04:03:38.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:38 smithi132 ceph-mon[23807]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:03:38.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:38 smithi132 ceph-mon[23807]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T04:03:38.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:38 smithi132 ceph-mon[23807]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:03:39.392 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:39 smithi132 ceph-mon[23807]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:03:39.486 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:39 smithi078 ceph-mon[22905]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:03:39.486 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:39 smithi078 ceph-mon[27629]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:03:41.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:41 smithi132 ceph-mon[23807]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:03:41.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:41 smithi078 ceph-mon[22905]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:03:41.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:41 smithi078 ceph-mon[27629]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:03:42.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:42 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:42 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:42 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:42 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:42 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:03:42.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:42 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:03:42.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:42 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:42 smithi132 ceph-mon[23807]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:42.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:42 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:42.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:03:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:03:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[27629]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:42.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:03:42.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:03:42.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:03:42.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[22905]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:42.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:42 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:45.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:45 smithi132 ceph-mon[23807]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:03:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:45 smithi078 ceph-mon[22905]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:03:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:45 smithi078 ceph-mon[27629]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:03:47.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:47 smithi132 ceph-mon[23807]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:03:47.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:47 smithi078 ceph-mon[22905]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:03:47.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:47 smithi078 ceph-mon[27629]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:03:49.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:49 smithi132 ceph-mon[23807]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:49.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:49 smithi078 ceph-mon[22905]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:49.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:49 smithi078 ceph-mon[27629]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:49.513 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:03:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:03:49] "GET /metrics HTTP/1.1" 200 38066 "" "Prometheus/2.33.4" 2024-05-09T04:03:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:50 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:03:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:50 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:03:50.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:50 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:03:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:51 smithi132 ceph-mon[23807]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:51 smithi078 ceph-mon[22905]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:51.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:51 smithi078 ceph-mon[27629]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:53 smithi132 ceph-mon[23807]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:53 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:53.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:53 smithi078 ceph-mon[22905]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:53.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:53 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:53.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:53 smithi078 ceph-mon[27629]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:53.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:53 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:03:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:55 smithi132 ceph-mon[23807]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:55.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:55 smithi078 ceph-mon[22905]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:55.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:55 smithi078 ceph-mon[27629]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:57.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:57 smithi132 ceph-mon[23807]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:57.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:57 smithi078 ceph-mon[22905]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:57.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:57 smithi078 ceph-mon[27629]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:03:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:03:59 smithi132 ceph-mon[23807]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:03:59 smithi078 ceph-mon[22905]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:03:59 smithi078 ceph-mon[27629]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:03:59.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:03:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:03:59] "GET /metrics HTTP/1.1" 200 38064 "" "Prometheus/2.33.4" 2024-05-09T04:04:01.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:01 smithi132 ceph-mon[23807]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:01.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:01 smithi078 ceph-mon[22905]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:01.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:01 smithi078 ceph-mon[27629]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:03 smithi132 ceph-mon[23807]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:03 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:03 smithi078 ceph-mon[22905]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:03 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:03.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:03 smithi078 ceph-mon[27629]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:03.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:03 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:05.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:05 smithi132 ceph-mon[23807]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:05.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:05 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:04:05.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:05 smithi078 ceph-mon[22905]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:05.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:05 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:04:05.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:05 smithi078 ceph-mon[27629]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:05.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:05 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:04:07.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:07 smithi132 ceph-mon[23807]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:07.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:07 smithi078 ceph-mon[22905]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:07.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:07 smithi078 ceph-mon[27629]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:09.314 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:09 smithi078 ceph-mon[22905]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:09.315 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:09 smithi078 ceph-mon[27629]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:09.315 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:04:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:04:09] "GET /metrics HTTP/1.1" 200 38061 "" "Prometheus/2.33.4" 2024-05-09T04:04:09.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:09 smithi132 ceph-mon[23807]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:10.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:10 smithi132 ceph-mon[23807]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:10 smithi078 ceph-mon[22905]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:10 smithi078 ceph-mon[27629]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:12.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:12 smithi132 ceph-mon[23807]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:12.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:12 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:12 smithi078 ceph-mon[22905]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:12 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:12.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:12 smithi078 ceph-mon[27629]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:12.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:12 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:15 smithi132 ceph-mon[23807]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:15.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:15 smithi078 ceph-mon[22905]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:15 smithi078 ceph-mon[27629]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:17.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:17 smithi132 ceph-mon[23807]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:17.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:17 smithi078 ceph-mon[22905]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:17.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:17 smithi078 ceph-mon[27629]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:19.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:19 smithi132 ceph-mon[23807]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:19.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:19 smithi078 ceph-mon[27629]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:19.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:19 smithi078 ceph-mon[22905]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:19.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:04:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:04:19] "GET /metrics HTTP/1.1" 200 38061 "" "Prometheus/2.33.4" 2024-05-09T04:04:20.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:04:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:04:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:04:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:21 smithi078 ceph-mon[22905]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:21.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:21 smithi078 ceph-mon[27629]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:21.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:21 smithi132 ceph-mon[23807]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:23.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:23 smithi132 ceph-mon[23807]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:23.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:23 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:23.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:23 smithi078 ceph-mon[22905]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:23.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:23 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:23.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:23 smithi078 ceph-mon[27629]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:23.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:23 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:25 smithi132 ceph-mon[23807]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:25 smithi078 ceph-mon[22905]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:25.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:25 smithi078 ceph-mon[27629]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:27.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:27 smithi132 ceph-mon[23807]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:27.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:27 smithi078 ceph-mon[22905]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:27.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:27 smithi078 ceph-mon[27629]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:29.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:29 smithi132 ceph-mon[23807]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:29.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:29 smithi078 ceph-mon[22905]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:29.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:04:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:04:29] "GET /metrics HTTP/1.1" 200 38059 "" "Prometheus/2.33.4" 2024-05-09T04:04:29.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:29 smithi078 ceph-mon[27629]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:31.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:31 smithi132 ceph-mon[23807]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:31 smithi078 ceph-mon[22905]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:31 smithi078 ceph-mon[27629]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:33.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:33 smithi132 ceph-mon[23807]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:33.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:33 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:33.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:33 smithi078 ceph-mon[27629]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:33.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:33 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:33 smithi078 ceph-mon[22905]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:33 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:35.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:35 smithi132 ceph-mon[23807]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:35.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:35 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:04:35.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:35 smithi078 ceph-mon[22905]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:35.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:35 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:04:35.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:35 smithi078 ceph-mon[27629]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:35.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:35 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:04:37.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:37 smithi132 ceph-mon[23807]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:37.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:37 smithi078 ceph-mon[22905]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:37.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:37 smithi078 ceph-mon[27629]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:39.314 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:39 smithi078 ceph-mon[22905]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:39.314 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:39 smithi078 ceph-mon[27629]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:39.314 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:04:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:04:39] "GET /metrics HTTP/1.1" 200 38052 "" "Prometheus/2.33.4" 2024-05-09T04:04:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:39 smithi132 ceph-mon[23807]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:41.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:41 smithi132 ceph-mon[23807]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:41.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:41 smithi078 ceph-mon[22905]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:41.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:41 smithi078 ceph-mon[27629]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:42.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:42 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:04:42.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:42 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:42.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:42 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:42.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:42 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:42.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:42 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:42 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:04:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:42 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:42 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:42 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:42 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:42 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:04:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:42 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:42 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:42 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:42 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:43 smithi132 ceph-mon[23807]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:43 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:43 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:04:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:43 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:04:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:43 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:43 smithi078 ceph-mon[22905]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:43 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:43 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:04:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:43 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:04:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:43 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:43 smithi078 ceph-mon[27629]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:43 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:43 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:04:43.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:43 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:04:43.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:43 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:04:45.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:45 smithi132 ceph-mon[23807]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:45 smithi078 ceph-mon[22905]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:45 smithi078 ceph-mon[27629]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:47.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:47 smithi132 ceph-mon[23807]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:47.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:47 smithi078 ceph-mon[22905]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:47.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:47 smithi078 ceph-mon[27629]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:49.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:49 smithi132 ceph-mon[23807]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:49.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:49 smithi078 ceph-mon[22905]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:49.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:49 smithi078 ceph-mon[27629]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:49.513 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:04:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:04:49] "GET /metrics HTTP/1.1" 200 38052 "" "Prometheus/2.33.4" 2024-05-09T04:04:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:50 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:04:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:50 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:04:50.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:50 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:04:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:51 smithi132 ceph-mon[23807]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:51 smithi078 ceph-mon[22905]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:51.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:51 smithi078 ceph-mon[27629]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:52 smithi132 ceph-mon[23807]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:52.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:52 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:52.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:52 smithi078 ceph-mon[22905]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:52.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:52 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:52.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:52 smithi078 ceph-mon[27629]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:52.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:52 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:04:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:55 smithi132 ceph-mon[23807]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:55.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:55 smithi078 ceph-mon[22905]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:55.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:55 smithi078 ceph-mon[27629]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:57.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:57 smithi132 ceph-mon[23807]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:57.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:57 smithi078 ceph-mon[22905]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:57.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:57 smithi078 ceph-mon[27629]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:04:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:04:59 smithi132 ceph-mon[23807]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:04:59 smithi078 ceph-mon[22905]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:04:59 smithi078 ceph-mon[27629]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:04:59.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:04:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:04:59] "GET /metrics HTTP/1.1" 200 38056 "" "Prometheus/2.33.4" 2024-05-09T04:05:01.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:01 smithi132 ceph-mon[23807]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:01.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:01 smithi078 ceph-mon[22905]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:01.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:01 smithi078 ceph-mon[27629]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:03 smithi132 ceph-mon[23807]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:03 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:03 smithi078 ceph-mon[22905]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:03 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:03.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:03 smithi078 ceph-mon[27629]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:03.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:03 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:05.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:05 smithi132 ceph-mon[23807]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:05.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:05 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:05:05.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:05 smithi078 ceph-mon[22905]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:05.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:05 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:05:05.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:05 smithi078 ceph-mon[27629]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:05.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:05 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:05:07.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:07 smithi132 ceph-mon[23807]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:07.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:07 smithi078 ceph-mon[22905]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:07.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:07 smithi078 ceph-mon[27629]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:09.314 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:09 smithi078 ceph-mon[27629]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:09.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:09 smithi078 ceph-mon[22905]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:09.315 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:05:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:05:09] "GET /metrics HTTP/1.1" 200 38060 "" "Prometheus/2.33.4" 2024-05-09T04:05:09.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:09 smithi132 ceph-mon[23807]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:11.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:11 smithi132 ceph-mon[23807]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:11.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:11 smithi078 ceph-mon[22905]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:11.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:11 smithi078 ceph-mon[27629]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:13.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:13 smithi132 ceph-mon[23807]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:13.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:13 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:13.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:13 smithi078 ceph-mon[22905]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:13.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:13 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:13.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:13 smithi078 ceph-mon[27629]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:13.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:13 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:15 smithi132 ceph-mon[23807]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:15.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:15 smithi078 ceph-mon[22905]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:15 smithi078 ceph-mon[27629]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:17.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:17 smithi132 ceph-mon[23807]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:17.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:17 smithi078 ceph-mon[22905]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:17.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:17 smithi078 ceph-mon[27629]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:19.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:19 smithi132 ceph-mon[23807]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:19.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:19 smithi078 ceph-mon[22905]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:19.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:19 smithi078 ceph-mon[27629]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:19.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:05:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:05:19] "GET /metrics HTTP/1.1" 200 38060 "" "Prometheus/2.33.4" 2024-05-09T04:05:20.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:05:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:05:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:05:21.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:21 smithi132 ceph-mon[23807]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:21 smithi078 ceph-mon[22905]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:21.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:21 smithi078 ceph-mon[27629]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:23.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:23 smithi132 ceph-mon[23807]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:23.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:23 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:23.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:23 smithi078 ceph-mon[22905]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:23.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:23 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:23.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:23 smithi078 ceph-mon[27629]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:23.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:23 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:25 smithi132 ceph-mon[23807]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:25.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:25 smithi078 ceph-mon[22905]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:25.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:25 smithi078 ceph-mon[27629]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:27.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:27 smithi132 ceph-mon[23807]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:27.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:27 smithi078 ceph-mon[22905]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:27.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:27 smithi078 ceph-mon[27629]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:29.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:29 smithi132 ceph-mon[23807]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:29.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:29 smithi078 ceph-mon[22905]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:29.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:29 smithi078 ceph-mon[27629]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:29.513 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:05:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:05:29] "GET /metrics HTTP/1.1" 200 38061 "" "Prometheus/2.33.4" 2024-05-09T04:05:31.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:31 smithi132 ceph-mon[23807]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:31 smithi078 ceph-mon[22905]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:31 smithi078 ceph-mon[27629]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:33.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:33 smithi132 ceph-mon[23807]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:33.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:33 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:33 smithi078 ceph-mon[22905]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:33 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:33.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:33 smithi078 ceph-mon[27629]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:33.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:33 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:35.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:35 smithi132 ceph-mon[23807]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:35.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:35 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:05:35.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:35 smithi078 ceph-mon[22905]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:35.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:35 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:05:35.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:35 smithi078 ceph-mon[27629]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:35.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:35 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:05:37.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:37 smithi132 ceph-mon[23807]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:37.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:37 smithi078 ceph-mon[22905]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:37.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:37 smithi078 ceph-mon[27629]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:39.314 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:39 smithi078 ceph-mon[22905]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:39.315 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:39 smithi078 ceph-mon[27629]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:39.315 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:05:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:05:39] "GET /metrics HTTP/1.1" 200 38069 "" "Prometheus/2.33.4" 2024-05-09T04:05:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:39 smithi132 ceph-mon[23807]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:41.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:41 smithi132 ceph-mon[23807]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:41.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:41 smithi078 ceph-mon[22905]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:41.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:41 smithi078 ceph-mon[27629]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:43 smithi132 ceph-mon[23807]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:43 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:43 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:05:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:43 smithi078 ceph-mon[22905]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:43 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:43.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:43 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:05:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:43 smithi078 ceph-mon[27629]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:43 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:43.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:43 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:05:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:44 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:05:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:44 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:05:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:44 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:05:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:44 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:05:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:44 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:05:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:44 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:05:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:44 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:05:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:44 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:05:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:44 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' 2024-05-09T04:05:45.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:45 smithi132 ceph-mon[23807]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:45 smithi078 ceph-mon[22905]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:45 smithi078 ceph-mon[27629]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:47.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:47 smithi132 ceph-mon[23807]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:47.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:47 smithi078 ceph-mon[22905]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:47.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:47 smithi078 ceph-mon[27629]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:49.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:49 smithi132 ceph-mon[23807]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:49.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:49 smithi078 ceph-mon[22905]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:49.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:49 smithi078 ceph-mon[27629]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:49.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:05:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:05:49] "GET /metrics HTTP/1.1" 200 38069 "" "Prometheus/2.33.4" 2024-05-09T04:05:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:50 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:05:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:50 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:05:50.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:50 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:05:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:51 smithi132 ceph-mon[23807]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:51.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:51 smithi078 ceph-mon[27629]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:51 smithi078 ceph-mon[22905]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:53 smithi132 ceph-mon[23807]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:53 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:53.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:53 smithi078 ceph-mon[22905]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:53.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:53 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:53.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:53 smithi078 ceph-mon[27629]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:53.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:53 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:05:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:55 smithi132 ceph-mon[23807]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:55.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:55 smithi078 ceph-mon[27629]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:55.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:55 smithi078 ceph-mon[22905]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:57.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:57 smithi132 ceph-mon[23807]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:57.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:57 smithi078 ceph-mon[22905]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:57.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:57 smithi078 ceph-mon[27629]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:05:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:05:59 smithi132 ceph-mon[23807]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:05:59 smithi078 ceph-mon[22905]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:05:59 smithi078 ceph-mon[27629]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:05:59.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:05:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:05:59] "GET /metrics HTTP/1.1" 200 38067 "" "Prometheus/2.33.4" 2024-05-09T04:06:01.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:01 smithi132 ceph-mon[23807]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:01.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:01 smithi078 ceph-mon[22905]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:01.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:01 smithi078 ceph-mon[27629]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:03 smithi132 ceph-mon[23807]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:03.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:03 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:03 smithi078 ceph-mon[22905]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:03.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:03 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:03.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:03 smithi078 ceph-mon[27629]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:03.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:03 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:05.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:05 smithi132 ceph-mon[23807]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:05.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:05 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:05.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:05 smithi078 ceph-mon[22905]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:05.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:05 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:05.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:05 smithi078 ceph-mon[27629]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:05.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:05 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:07.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:07 smithi132 ceph-mon[23807]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:07.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:07 smithi078 ceph-mon[22905]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:07.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:07 smithi078 ceph-mon[27629]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:09.314 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:09 smithi078 ceph-mon[27629]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:09.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:09 smithi078 ceph-mon[22905]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:09.315 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:06:09] "GET /metrics HTTP/1.1" 200 38066 "" "Prometheus/2.33.4" 2024-05-09T04:06:09.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:09 smithi132 ceph-mon[23807]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:11.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:11 smithi132 ceph-mon[23807]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:11.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:11 smithi078 ceph-mon[22905]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:11.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:11 smithi078 ceph-mon[27629]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:13.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:13 smithi132 ceph-mon[23807]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:13.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:13 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:13.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:13 smithi078 ceph-mon[22905]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:13.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:13 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:13.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:13 smithi078 ceph-mon[27629]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:13.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:13 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:15.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:15 smithi132 ceph-mon[23807]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:15.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:15 smithi078 ceph-mon[22905]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:15.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:15 smithi078 ceph-mon[27629]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:17.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:17 smithi132 ceph-mon[23807]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:17.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:17 smithi078 ceph-mon[22905]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:17.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:17 smithi078 ceph-mon[27629]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:19.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:19 smithi132 ceph-mon[23807]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:19.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:19 smithi078 ceph-mon[22905]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:19.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:19 smithi078 ceph-mon[27629]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:19.513 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:06:19] "GET /metrics HTTP/1.1" 200 38066 "" "Prometheus/2.33.4" 2024-05-09T04:06:20.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:20 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:20.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:20 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:20 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:21.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:21 smithi132 ceph-mon[23807]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:21 smithi078 ceph-mon[22905]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:21.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:21 smithi078 ceph-mon[27629]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:22.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T04:06:23.414 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:23 smithi078 ceph-mon[27629]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:23.414 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:23 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:23.414 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:23 smithi078 ceph-mon[22905]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:23.414 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:23 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:23.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:23 smithi132 ceph-mon[23807]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:23.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:23 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:24.407 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:24 smithi078 ceph-mon[27629]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:24.407 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:24 smithi078 ceph-mon[22905]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:24.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:24 smithi132 ceph-mon[23807]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:24.852 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (9m) 2m ago 17m 24.1M - 0.23.0 ba2b418f427c f44cb50f5f35 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (8m) 2m ago 16m 53.5M - 8.3.5 dad864ee21e9 33b2a450dbe1 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (2m) 2m ago 16m 48.5M - 3.5 e1d6a67b021e 6b2e2e14b01e 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283 running (11m) 2m ago 21m 462M - 19.0.0-3631-gc66c553a af4a4df59e0a 4a3f6038cf25 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (6m) 2m ago 23m 508M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (23m) 2m ago 23m 62.4M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (21m) 2m ago 21m 49.9M 2048M 17.2.0 e1d6a67b021e a71bae52559d 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (22m) 2m ago 22m 50.5M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (9m) 2m ago 17m 17.9M - 1.3.1 1dbe0e931976 4a3cc1414851 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (9m) 2m ago 17m 20.2M - 1.3.1 1dbe0e931976 666630a53114 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (21m) 2m ago 21m 53.2M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (20m) 2m ago 20m 56.7M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (20m) 2m ago 20m 50.2M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:06:24.853 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (20m) 2m ago 20m 53.1M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:06:24.854 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (19m) 2m ago 19m 54.8M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:06:24.854 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (19m) 2m ago 19m 52.1M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:06:24.854 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (18m) 2m ago 18m 51.1M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:06:24.854 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (18m) 2m ago 18m 55.5M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:06:24.854 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (2m) 2m ago 17m 42.3M - 2.33.4 514e6a882f6e a1ad9e190e07 2024-05-09T04:06:24.854 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (16m) 2m ago 16m 76.0M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:06:24.854 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (16m) 2m ago 16m 77.3M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:06:25.462 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:25 smithi078 ceph-mon[27629]: from='client.15345 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:06:25.462 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:25 smithi078 ceph-mon[22905]: from='client.15345 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:06:25.485 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions' 2024-05-09T04:06:25.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:25 smithi132 ceph-mon[23807]: from='client.15345 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:06:26.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:26 smithi078 ceph-mon[22905]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:26 smithi078 ceph-mon[27629]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:26.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:26 smithi132 ceph-mon[23807]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:27.443 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:06:27.443 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:06:27.443 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-09T04:06:27.443 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:06:27.443 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:06:27.443 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:06:27.443 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:06:27.443 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:06:27.443 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T04:06:27.443 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:06:27.444 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": {}, 2024-05-09T04:06:27.444 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:06:27.444 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:06:27.444 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:06:27.444 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:06:27.444 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-09T04:06:27.444 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:06:27.444 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:06:27.444 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:06:27.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:27 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3796885776' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:06:27.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:27 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3796885776' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:06:27.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:27 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3796885776' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:06:28.095 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph -s' 2024-05-09T04:06:29.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:29 smithi078 ceph-mon[27629]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:29.359 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:29 smithi078 ceph-mon[22905]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:29.359 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:06:29] "GET /metrics HTTP/1.1" 200 38064 "" "Prometheus/2.33.4" 2024-05-09T04:06:29.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:29 smithi132 ceph-mon[23807]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:30.111 INFO:teuthology.orchestra.run.smithi078.stdout: cluster: 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: id: 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: health: HEALTH_OK 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: services: 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: mon: 3 daemons, quorum a,c,b (age 21m) 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: mgr: y(active, since 3m), standbys: x 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: data: 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: pools: 6 pools, 161 pgs 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: objects: 241 objects, 585 KiB 2024-05-09T04:06:30.112 INFO:teuthology.orchestra.run.smithi078.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:30.113 INFO:teuthology.orchestra.run.smithi078.stdout: pgs: 161 active+clean 2024-05-09T04:06:30.113 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T04:06:30.113 INFO:teuthology.orchestra.run.smithi078.stdout: io: 2024-05-09T04:06:30.113 INFO:teuthology.orchestra.run.smithi078.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-09T04:06:30.113 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-09T04:06:30.406 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:30 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/13223416' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T04:06:30.406 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:30 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/13223416' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T04:06:30.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:30 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/13223416' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-09T04:06:30.713 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph health detail' 2024-05-09T04:06:31.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:31 smithi078 ceph-mon[27629]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:31.343 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:31 smithi078 ceph-mon[22905]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:31.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:31 smithi132 ceph-mon[23807]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:32.780 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-05-09T04:06:33.371 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:33 smithi078 ceph-mon[27629]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:33.371 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:33 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:33.371 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:33 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2812530952' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:06:33.371 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:33 smithi078 ceph-mon[22905]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:33.371 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:33 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:33.371 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:33 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2812530952' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:06:33.394 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-05-09T04:06:33.510 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:33 smithi132 ceph-mon[23807]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:33.510 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:33 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:33.510 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:33 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2812530952' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:06:35.408 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:06:35.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:35 smithi132 ceph-mon[23807]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:35.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:35 smithi132 ceph-mon[23807]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:35.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:35 smithi078 ceph-mon[27629]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:35.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:35 smithi078 ceph-mon[27629]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:35.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:35 smithi078 ceph-mon[22905]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:06:35.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:35 smithi078 ceph-mon[22905]: from='mgr.25129 172.21.15.78:0/520020770' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:36.085 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph mgr fail' 2024-05-09T04:06:36.402 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:36 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2870983648' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:06:36.403 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:36 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2870983648' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:06:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:36 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2870983648' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:06:37.374 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:37 smithi078 ceph-mon[27629]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:37.375 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:37 smithi078 ceph-mon[22905]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:37.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:37 smithi132 ceph-mon[23807]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:06:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:38 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1198384961' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T04:06:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:38 smithi132 ceph-mon[23807]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T04:06:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:38 smithi132 ceph-mon[23807]: osdmap e105: 8 total, 8 up, 8 in 2024-05-09T04:06:38.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:04:06:38] ENGINE Bus STOPPING 2024-05-09T04:06:38.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:04:06:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-09T04:06:38.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:04:06:38] ENGINE Bus STOPPED 2024-05-09T04:06:38.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:04:06:38] ENGINE Bus STARTING 2024-05-09T04:06:38.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:04:06:38] ENGINE Serving on http://:::9283 2024-05-09T04:06:38.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:04:06:38] ENGINE Bus STARTED 2024-05-09T04:06:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:38 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1198384961' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T04:06:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:38 smithi078 ceph-mon[22905]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T04:06:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:38 smithi078 ceph-mon[22905]: osdmap e105: 8 total, 8 up, 8 in 2024-05-09T04:06:38.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:38 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:06:38.185+0000 7fb9c199a700 -1 mgr handle_mgr_map I was active but no longer am 2024-05-09T04:06:38.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:38 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ignoring --setuser ceph since I am not root 2024-05-09T04:06:38.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:38 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ignoring --setgroup ceph since I am not root 2024-05-09T04:06:38.513 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:38 smithi078 ceph-mgr[61747]: -- 172.21.15.78:0/3864742943 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5604c7cb85a0 con 0x5604c7c8d000 2024-05-09T04:06:38.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:38 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1198384961' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T04:06:38.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:38 smithi078 ceph-mon[27629]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-09T04:06:38.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:38 smithi078 ceph-mon[27629]: osdmap e105: 8 total, 8 up, 8 in 2024-05-09T04:06:38.883 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'sleep 180' 2024-05-09T04:06:39.462 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-09T04:06:39.462 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: mgrmap e33: x(active, starting, since 0.0473163s) 2024-05-09T04:06:39.462 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:06:39.462 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:06:39.462 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: Manager daemon x is now available 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:06:39.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:39.464 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:06:39.464 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:06:39.464 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-09T04:06:39.464 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-09T04:06:39.464 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-09T04:06:39.464 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: mgrmap e33: x(active, starting, since 0.0473163s) 2024-05-09T04:06:39.464 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:06:39.464 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:06:39.464 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:06:39.464 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T04:06:39.464 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: Manager daemon x is now available 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:06:39.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:06:39.466 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-09T04:06:39.466 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:39 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-09T04:06:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-09T04:06:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: mgrmap e33: x(active, starting, since 0.0473163s) 2024-05-09T04:06:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:06:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:06:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:06:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: Manager daemon x is now available 2024-05-09T04:06:39.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:06:39.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:39.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:06:39.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:06:39.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-09T04:06:39.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:39 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-09T04:06:40.400 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:40 smithi132 ceph-mon[23807]: [09/May/2024:04:06:38] ENGINE Bus STARTING 2024-05-09T04:06:40.401 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:40 smithi132 ceph-mon[23807]: [09/May/2024:04:06:38] ENGINE Serving on http://172.21.15.132:8765 2024-05-09T04:06:40.401 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:40 smithi132 ceph-mon[23807]: [09/May/2024:04:06:39] ENGINE Serving on https://172.21.15.132:7150 2024-05-09T04:06:40.401 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:40 smithi132 ceph-mon[23807]: [09/May/2024:04:06:39] ENGINE Bus STARTED 2024-05-09T04:06:40.401 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:40 smithi132 ceph-mon[23807]: mgrmap e34: x(active, since 1.05111s) 2024-05-09T04:06:40.401 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:40 smithi132 ceph-mon[23807]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:40.493 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:40 smithi078 ceph-mon[27629]: [09/May/2024:04:06:38] ENGINE Bus STARTING 2024-05-09T04:06:40.493 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:40 smithi078 ceph-mon[27629]: [09/May/2024:04:06:38] ENGINE Serving on http://172.21.15.132:8765 2024-05-09T04:06:40.493 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:40 smithi078 ceph-mon[27629]: [09/May/2024:04:06:39] ENGINE Serving on https://172.21.15.132:7150 2024-05-09T04:06:40.493 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:40 smithi078 ceph-mon[27629]: [09/May/2024:04:06:39] ENGINE Bus STARTED 2024-05-09T04:06:40.493 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:40 smithi078 ceph-mon[27629]: mgrmap e34: x(active, since 1.05111s) 2024-05-09T04:06:40.493 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:40 smithi078 ceph-mon[27629]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:40.493 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:40 smithi078 ceph-mon[22905]: [09/May/2024:04:06:38] ENGINE Bus STARTING 2024-05-09T04:06:40.493 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:40 smithi078 ceph-mon[22905]: [09/May/2024:04:06:38] ENGINE Serving on http://172.21.15.132:8765 2024-05-09T04:06:40.493 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:40 smithi078 ceph-mon[22905]: [09/May/2024:04:06:39] ENGINE Serving on https://172.21.15.132:7150 2024-05-09T04:06:40.493 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:40 smithi078 ceph-mon[22905]: [09/May/2024:04:06:39] ENGINE Bus STARTED 2024-05-09T04:06:40.493 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:40 smithi078 ceph-mon[22905]: mgrmap e34: x(active, since 1.05111s) 2024-05-09T04:06:40.494 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:40 smithi078 ceph-mon[22905]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:41.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:41 smithi132 ceph-mon[23807]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:41.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:41 smithi078 ceph-mon[22905]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:41.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:41 smithi078 ceph-mon[27629]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:42.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:42 smithi132 ceph-mon[23807]: mgrmap e35: x(active, since 3s) 2024-05-09T04:06:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:42 smithi078 ceph-mon[22905]: mgrmap e35: x(active, since 3s) 2024-05-09T04:06:42.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:42 smithi078 ceph-mon[27629]: mgrmap e35: x(active, since 3s) 2024-05-09T04:06:43.439 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:43 smithi078 ceph-mon[22905]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:43.439 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:43 smithi078 ceph-mon[22905]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:43.439 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:43 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:43.439 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:43 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:43.439 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:43 smithi078 ceph-mon[27629]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:43.440 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:43 smithi078 ceph-mon[27629]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:43.440 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:43 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:43.440 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:43 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:43 smithi132 ceph-mon[23807]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:43 smithi132 ceph-mon[23807]: from='client.25228 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:06:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:43 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:43.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:43 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:45.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:45 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:45.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:45 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:45.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:45 smithi132 ceph-mon[23807]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:45.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:45 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:45.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:45 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:45.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:45 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:45 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:45 smithi078 ceph-mon[22905]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:45 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:45 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:45 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:45 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:45 smithi078 ceph-mon[27629]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:06:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:45 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:45 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:46.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:45 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:06:45] ENGINE Bus STARTING 2024-05-09T04:06:46.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:45 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: CherryPy Checker: 2024-05-09T04:06:46.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:45 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: The Application mounted at '' has an empty config. 2024-05-09T04:06:46.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:45 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:06:46.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:45 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:06:45] ENGINE Serving on http://:::9283 2024-05-09T04:06:46.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:45 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:06:45] ENGINE Bus STARTED 2024-05-09T04:06:47.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:06:47.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:06:47.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T04:06:47.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T04:06:47.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: Standby manager daemon y started 2024-05-09T04:06:47.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-09T04:06:47.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:06:47.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-09T04:06:47.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:06:47.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:06:47.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:06:47.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:06:47.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:06:47.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T04:06:47.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:06:47.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:46 smithi132 ceph-mon[23807]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:06:47.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:06:47.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:06:47.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T04:06:47.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T04:06:47.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: Standby manager daemon y started 2024-05-09T04:06:47.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-09T04:06:47.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:06:47.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-09T04:06:47.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:06:47.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:06:47.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:06:47.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:06:47.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:06:47.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T04:06:47.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:06:47.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[22905]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:06:47.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:06:47.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:06:47.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T04:06:47.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T04:06:47.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: Standby manager daemon y started 2024-05-09T04:06:47.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-09T04:06:47.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:06:47.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-09T04:06:47.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:06:47.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:06:47.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:06:47.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:06:47.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:06:47.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T04:06:47.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:06:47.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:46 smithi078 ceph-mon[27629]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:06:47.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[27629]: mgrmap e36: x(active, since 8s), standbys: y 2024-05-09T04:06:47.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T04:06:47.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[27629]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:06:47.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:06:47.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:06:47.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:06:47.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[27629]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:06:47.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[22905]: mgrmap e36: x(active, since 8s), standbys: y 2024-05-09T04:06:47.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T04:06:47.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:47.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[22905]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:06:47.759 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:06:47.759 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:06:47.759 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:06:47.759 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:47 smithi078 ceph-mon[22905]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:06:48.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:47 smithi132 ceph-mon[23807]: mgrmap e36: x(active, since 8s), standbys: y 2024-05-09T04:06:48.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:47 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T04:06:48.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:47 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:48.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:47 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:48.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:47 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:48.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:47 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:48.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:47 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:48.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:47 smithi132 ceph-mon[23807]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:06:48.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:47 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:06:48.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:47 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:06:48.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:47 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:06:48.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:47 smithi132 ceph-mon[23807]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:06:49.166 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:06:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:06:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-09T04:06:49.441 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:49 smithi078 ceph-mon[22905]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T04:06:49.442 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:49 smithi078 ceph-mon[27629]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T04:06:49.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:49 smithi132 ceph-mon[23807]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T04:06:50.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T04:06:50.018+0000 7f6f7e760700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:06:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Creating ceph-iscsi config... 2024-05-09T04:06:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:06:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:06:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:06:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T04:06:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:06:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return _run_code(code, main_globals, None, 2024-05-09T04:06:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:06:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: exec(code, run_globals) 2024-05-09T04:06:50.505 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T04:06:50.506 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return self.event_loop.get_result(coro, timeout) 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return future.result(timeout) 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return self.__get_result() 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: raise self._exception 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: use_current_daemon_image=reconfig, 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Creating ceph-iscsi config... 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:06:50.507 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:06:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:06:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: Traceback (most recent call last): 2024-05-09T04:06:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:06:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: return _run_code(code, main_globals, None, 2024-05-09T04:06:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:06:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: exec(code, run_globals) 2024-05-09T04:06:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:06:50.508 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:06:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:06:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:06:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:06:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:06:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:06:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:50.509 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:06:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T04:06:50.019+0000 7f6f7e760700 -1 log_channel(cephadm) log [ERR] : Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:06:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:51 smithi132 ceph-mon[23807]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:06:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout: Creating ceph-iscsi config... 2024-05-09T04:06:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:06:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:06:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: exec(code, run_globals) 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:06:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: return future.result(timeout) 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: return self.__get_result() 2024-05-09T04:06:51.506 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T04:06:51.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: raise self._exception 2024-05-09T04:06:51.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T04:06:51.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: use_current_daemon_image=reconfig, 2024-05-09T04:06:51.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T04:06:51.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T04:06:51.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:06:51.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: Creating ceph-iscsi config... 2024-05-09T04:06:51.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:06:51.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:06:51.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:06:51.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:51.507 INFO:journalctl@ceph.mon.b.smithi132.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:51.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: Traceback (most recent call last): 2024-05-09T04:06:51.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:06:51.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:06:51.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:06:51.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: exec(code, run_globals) 2024-05-09T04:06:51.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:06:51.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:06:51.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:06:51.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:06:51.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:06:51.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:06:51.508 INFO:journalctl@ceph.mon.b.smithi132.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:06:51.509 INFO:journalctl@ceph.mon.b.smithi132.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:51.509 INFO:journalctl@ceph.mon.b.smithi132.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:51.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:51 smithi132 ceph-mon[23807]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T04:06:51.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:51 smithi132 ceph-mon[23807]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:06:51.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:51 smithi132 ceph-mon[23807]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:06:51.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:51 smithi132 ceph-mon[23807]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T04:06:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:51 smithi078 ceph-mon[22905]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:06:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T04:06:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:06:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:06:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:06:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: exec(code, run_globals) 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: return future.result(timeout) 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: return self.__get_result() 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: raise self._exception 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T04:06:51.514 INFO:journalctl@ceph.mon.a.smithi078.stdout: use_current_daemon_image=reconfig, 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: Traceback (most recent call last): 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: exec(code, run_globals) 2024-05-09T04:06:51.515 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:06:51.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:06:51.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:06:51.516 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:06:51.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:51 smithi078 ceph-mon[27629]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:06:51.516 INFO:journalctl@ceph.mon.c.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T04:06:51.516 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:06:51.516 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:06:51.516 INFO:journalctl@ceph.mon.c.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:06:51.516 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:51.516 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: exec(code, run_globals) 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:51.517 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T04:06:51.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1132, in _check_daemons 2024-05-09T04:06:51.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-09T04:06:51.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2464, in _daemon_action 2024-05-09T04:06:51.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-09T04:06:51.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 785, in wait_async 2024-05-09T04:06:51.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-09T04:06:51.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-09T04:06:51.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: return future.result(timeout) 2024-05-09T04:06:51.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-09T04:06:51.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: return self.__get_result() 2024-05-09T04:06:51.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-09T04:06:51.518 INFO:journalctl@ceph.mon.c.smithi078.stdout: raise self._exception 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1395, in _create_daemon 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: use_current_daemon_image=reconfig, 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1712, in _run_cephadm 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: Creating ceph-iscsi config... 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: Non-zero exit code 1 from systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: Traceback (most recent call last): 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:06:51.519 INFO:journalctl@ceph.mon.c.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:06:51.521 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:06:51.521 INFO:journalctl@ceph.mon.c.smithi078.stdout: exec(code, run_globals) 2024-05-09T04:06:51.521 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:06:51.521 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:06:51.521 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:06:51.521 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:06:51.522 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:06:51.522 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:06:51.522 INFO:journalctl@ceph.mon.a.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:06:51.522 INFO:journalctl@ceph.mon.a.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:51.522 INFO:journalctl@ceph.mon.a.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:51.523 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:51 smithi078 ceph-mon[22905]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T04:06:51.523 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:51 smithi078 ceph-mon[22905]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:06:51.523 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:51 smithi078 ceph-mon[22905]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:06:51.523 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:51 smithi078 ceph-mon[22905]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T04:06:51.523 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:06:51.523 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1123, in deploy_daemon 2024-05-09T04:06:51.523 INFO:journalctl@ceph.mon.c.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:06:51.523 INFO:journalctl@ceph.mon.c.smithi078.stdout: RuntimeError: Failed command: systemctl restart ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:06:51.523 INFO:journalctl@ceph.mon.c.smithi078.stdout: See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:06:51.523 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:51 smithi078 ceph-mon[27629]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T04:06:51.523 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:51 smithi078 ceph-mon[27629]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:06:51.523 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:51 smithi078 ceph-mon[27629]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:06:51.524 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:51 smithi078 ceph-mon[27629]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T04:06:53.477 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:53 smithi132 ceph-mon[23807]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:06:53.477 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 systemd[1]: Stopping Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:06:53.477 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 bash[55657]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus.a" found: no such container 2024-05-09T04:06:53.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:53 smithi078 ceph-mon[22905]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:06:53.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:53 smithi078 ceph-mon[27629]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:06:53.752 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:06:53.476Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-09T04:06:53.752 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:06:53.476Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-09T04:06:53.752 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:06:53.476Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-09T04:06:53.752 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:06:53.476Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-09T04:06:53.753 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:06:53.476Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-09T04:06:53.753 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:06:53.476Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-09T04:06:53.753 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:06:53.476Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-09T04:06:53.753 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:06:53.476Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-09T04:06:53.753 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:06:53.476Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-09T04:06:53.753 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:06:53.477Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-09T04:06:53.753 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:06:53.477Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-09T04:06:53.753 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[52745]: ts=2024-05-09T04:06:53.477Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-09T04:06:53.753 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 podman[55670]: 2024-05-09 04:06:53.486455493 +0000 UTC m=+0.105892520 container died a1ad9e190e07d75e0b959051c9e644506ed3e7071924ba61f38efeb5b1c7deb0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:06:54.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 podman[55670]: 2024-05-09 04:06:53.859743038 +0000 UTC m=+0.479180081 container cleanup a1ad9e190e07d75e0b959051c9e644506ed3e7071924ba61f38efeb5b1c7deb0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:06:54.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 bash[55670]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a 2024-05-09T04:06:54.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:53 smithi132 bash[55698]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus.a" found: no such container 2024-05-09T04:06:54.436 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:54 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:54.436 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 podman[55685]: 2024-05-09 04:06:54.143644912 +0000 UTC m=+0.659030829 container remove a1ad9e190e07d75e0b959051c9e644506ed3e7071924ba61f38efeb5b1c7deb0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:06:54.437 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-09T04:06:54.437 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 systemd[1]: Stopped Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:06:54.437 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service: Consumed 1.415s CPU time. 2024-05-09T04:06:54.437 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 systemd[1]: Starting Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:06:54.437 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 podman[55766]: 2024-05-09 04:06:54.285403317 +0000 UTC m=+0.019293743 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-09T04:06:54.437 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 podman[55766]: 2024-05-09 04:06:54.436460478 +0000 UTC m=+0.170350931 container create fef66365cfcd89d72ab19e17961f97006461318230af92408cb3ba925086fa62 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:06:54.706 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 podman[55766]: 2024-05-09 04:06:54.630135351 +0000 UTC m=+0.364025768 container init fef66365cfcd89d72ab19e17961f97006461318230af92408cb3ba925086fa62 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:06:54.706 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 podman[55766]: 2024-05-09 04:06:54.632828729 +0000 UTC m=+0.366719141 container start fef66365cfcd89d72ab19e17961f97006461318230af92408cb3ba925086fa62 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:06:54.706 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.656Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-09T04:06:54.706 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.656Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-09T04:06:54.706 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.656Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-09T04:06:54.706 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.656Z caller=main.go:518 level=info host_details="(Linux 5.14.0-444.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024 x86_64 smithi132 (none))" 2024-05-09T04:06:54.706 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.656Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-09T04:06:54.706 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.656Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-09T04:06:54.706 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.657Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-09T04:06:54.706 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.658Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-09T04:06:54.706 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.659Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-09T04:06:54.707 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.660Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-09T04:06:54.707 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.660Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=674.962µs 2024-05-09T04:06:54.707 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.660Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-09T04:06:54.707 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 bash[55766]: fef66365cfcd89d72ab19e17961f97006461318230af92408cb3ba925086fa62 2024-05-09T04:06:54.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:54 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:54.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:54 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:06:55.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 systemd[1]: Started Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:06:55.004 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:54.886Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-05-09T04:06:55.394 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:55 smithi132 ceph-mon[23807]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:06:55.394 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:55 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:55.394 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:55 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:55.394 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:55 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:06:55.394 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:55 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:06:55.395 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:55 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:55.140Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-05-09T04:06:55.395 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:55 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:55.360Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-05-09T04:06:55.395 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:55 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:55.363Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-05-09T04:06:55.395 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:55 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:55.363Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-05-09T04:06:55.395 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:55 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:55.364Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.334µs wal_replay_duration=703.16568ms total_replay_duration=703.892737ms 2024-05-09T04:06:55.395 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:55 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:55.368Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-09T04:06:55.395 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:55 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:55.368Z caller=main.go:947 level=info msg="TSDB started" 2024-05-09T04:06:55.395 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:55 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:55.368Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-09T04:06:55.395 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:55 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:55.394Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.36261ms db_storage=899ns remote_storage=1.604µs web_handler=442ns query_engine=796ns scrape=547.51µs scrape_sd=89.547µs notify=17.03µs notify_sd=16.444µs rules=24.310274ms 2024-05-09T04:06:55.395 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:06:55 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:06:55.394Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-09T04:06:55.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:55 smithi078 ceph-mon[27629]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:06:55.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:55 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:55.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:55 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:55.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:55 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:06:55.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:55 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:06:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:55 smithi078 ceph-mon[22905]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:06:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:55 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:55.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:55 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:55.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:55 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:06:55.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:55 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:06:56.388 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:56 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:06:56.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:56 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:06:56.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:56 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:06:57.292 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:57 smithi132 ceph-mon[23807]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:06:57.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:57 smithi078 ceph-mon[22905]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:06:57.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:57 smithi078 ceph-mon[27629]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-05-09T04:06:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:59 smithi132 ceph-mon[23807]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:06:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:59 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:59 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:59 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:06:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:59 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:06:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:06:59 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:59 smithi078 ceph-mon[22905]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:06:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:59 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:59 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:59 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:06:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:59 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:06:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:06:59 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:59 smithi078 ceph-mon[27629]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:06:59.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:59 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:59.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:59 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:06:59.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:59 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:06:59.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:59 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:06:59.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:06:59 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:00 smithi132 ceph-mon[23807]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:07:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:00 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:07:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:00 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:07:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:00 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:07:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:00 smithi132 ceph-mon[23807]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:07:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:00 smithi078 ceph-mon[22905]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:07:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:00 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:07:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:00 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:07:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:00 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:07:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:00 smithi078 ceph-mon[22905]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:07:00.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:00 smithi078 ceph-mon[27629]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:07:00.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:00 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:07:00.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:00 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:07:00.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:00 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:07:00.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:00 smithi078 ceph-mon[27629]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:07:01.455 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:01 smithi078 ceph-mon[22905]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:07:01.456 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:01 smithi078 ceph-mon[27629]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:07:01.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:01 smithi132 ceph-mon[23807]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:07:03.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:03 smithi078 ceph-mon[22905]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:07:03.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:03 smithi078 ceph-mon[27629]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:07:03.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:03 smithi132 ceph-mon[23807]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:07:05.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:07:04 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:07:04] "GET /metrics HTTP/1.1" 200 38068 "" "Prometheus/2.33.4" 2024-05-09T04:07:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:05 smithi132 ceph-mon[23807]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:07:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:05 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:05 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:05 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:07:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:05 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:07:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:05 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:07:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:05 smithi132 ceph-mon[23807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:07:05.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:05 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:07:05.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:05 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1167945729' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:07:05.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:05 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1321829037' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2716738216"}]: dispatch 2024-05-09T04:07:05.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[22905]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:07:05.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:05.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:05.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:07:05.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:07:05.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:07:05.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[22905]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:07:05.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:07:05.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1167945729' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:07:05.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1321829037' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2716738216"}]: dispatch 2024-05-09T04:07:05.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[27629]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:07:05.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:05.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:05.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:07:05.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:07:05.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:07:05.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[27629]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:07:05.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:07:05.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1167945729' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:07:05.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:05 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1321829037' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2716738216"}]: dispatch 2024-05-09T04:07:06.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:06 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1321829037' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2716738216"}]': finished 2024-05-09T04:07:06.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:06 smithi132 ceph-mon[23807]: osdmap e106: 8 total, 8 up, 8 in 2024-05-09T04:07:06.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:06 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/1479715279' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/39992791"}]: dispatch 2024-05-09T04:07:06.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:06 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/39992791"}]: dispatch 2024-05-09T04:07:06.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:06 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1321829037' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2716738216"}]': finished 2024-05-09T04:07:06.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:06 smithi078 ceph-mon[22905]: osdmap e106: 8 total, 8 up, 8 in 2024-05-09T04:07:06.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:06 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/1479715279' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/39992791"}]: dispatch 2024-05-09T04:07:06.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:06 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/39992791"}]: dispatch 2024-05-09T04:07:06.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:06 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1321829037' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2716738216"}]': finished 2024-05-09T04:07:06.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:06 smithi078 ceph-mon[27629]: osdmap e106: 8 total, 8 up, 8 in 2024-05-09T04:07:06.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:06 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/1479715279' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/39992791"}]: dispatch 2024-05-09T04:07:06.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:06 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/39992791"}]: dispatch 2024-05-09T04:07:07.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:07 smithi132 ceph-mon[23807]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:07:07.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:07 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/39992791"}]': finished 2024-05-09T04:07:07.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:07 smithi132 ceph-mon[23807]: osdmap e107: 8 total, 8 up, 8 in 2024-05-09T04:07:07.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:07 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3081865972' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2716738216"}]: dispatch 2024-05-09T04:07:07.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:07 smithi078 ceph-mon[22905]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:07:07.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:07 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/39992791"}]': finished 2024-05-09T04:07:07.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:07 smithi078 ceph-mon[22905]: osdmap e107: 8 total, 8 up, 8 in 2024-05-09T04:07:07.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:07 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3081865972' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2716738216"}]: dispatch 2024-05-09T04:07:07.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:07 smithi078 ceph-mon[27629]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:07:07.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:07 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/39992791"}]': finished 2024-05-09T04:07:07.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:07 smithi078 ceph-mon[27629]: osdmap e107: 8 total, 8 up, 8 in 2024-05-09T04:07:07.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:07 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3081865972' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2716738216"}]: dispatch 2024-05-09T04:07:08.682 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:08 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3081865972' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2716738216"}]': finished 2024-05-09T04:07:08.682 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:08 smithi078 ceph-mon[27629]: osdmap e108: 8 total, 8 up, 8 in 2024-05-09T04:07:08.682 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:08 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/834697744' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4119666521"}]: dispatch 2024-05-09T04:07:08.682 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:08 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:07:08.683 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:08 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3081865972' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2716738216"}]': finished 2024-05-09T04:07:08.683 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:08 smithi078 ceph-mon[22905]: osdmap e108: 8 total, 8 up, 8 in 2024-05-09T04:07:08.683 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:08 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/834697744' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4119666521"}]: dispatch 2024-05-09T04:07:08.683 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:08 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:07:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:08 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3081865972' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2716738216"}]': finished 2024-05-09T04:07:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:08 smithi132 ceph-mon[23807]: osdmap e108: 8 total, 8 up, 8 in 2024-05-09T04:07:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:08 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/834697744' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4119666521"}]: dispatch 2024-05-09T04:07:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:08 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:07:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:09 smithi132 ceph-mon[23807]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:07:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:09 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/834697744' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4119666521"}]': finished 2024-05-09T04:07:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:09 smithi132 ceph-mon[23807]: osdmap e109: 8 total, 8 up, 8 in 2024-05-09T04:07:09.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:09 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/64053184' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4040605258"}]: dispatch 2024-05-09T04:07:09.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:09 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4040605258"}]: dispatch 2024-05-09T04:07:09.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:09 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:09.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:09 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:09.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:09 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:07:09.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:09 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:07:09.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:09 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[22905]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:07:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/834697744' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4119666521"}]': finished 2024-05-09T04:07:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[22905]: osdmap e109: 8 total, 8 up, 8 in 2024-05-09T04:07:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/64053184' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4040605258"}]: dispatch 2024-05-09T04:07:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4040605258"}]: dispatch 2024-05-09T04:07:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:07:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:07:09.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:09.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[27629]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:07:09.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/834697744' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4119666521"}]': finished 2024-05-09T04:07:09.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[27629]: osdmap e109: 8 total, 8 up, 8 in 2024-05-09T04:07:09.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/64053184' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4040605258"}]: dispatch 2024-05-09T04:07:09.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4040605258"}]: dispatch 2024-05-09T04:07:09.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:09.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:09.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:07:09.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:07:09.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:09 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:07:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:10 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4040605258"}]': finished 2024-05-09T04:07:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:10 smithi132 ceph-mon[23807]: osdmap e110: 8 total, 8 up, 8 in 2024-05-09T04:07:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:10 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3372511896' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3199996049"}]: dispatch 2024-05-09T04:07:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:10 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3199996049"}]: dispatch 2024-05-09T04:07:10.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:10 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4040605258"}]': finished 2024-05-09T04:07:10.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:10 smithi078 ceph-mon[22905]: osdmap e110: 8 total, 8 up, 8 in 2024-05-09T04:07:10.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:10 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3372511896' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3199996049"}]: dispatch 2024-05-09T04:07:10.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:10 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3199996049"}]: dispatch 2024-05-09T04:07:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:10 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4040605258"}]': finished 2024-05-09T04:07:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:10 smithi078 ceph-mon[27629]: osdmap e110: 8 total, 8 up, 8 in 2024-05-09T04:07:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:10 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3372511896' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3199996049"}]: dispatch 2024-05-09T04:07:10.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:10 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3199996049"}]: dispatch 2024-05-09T04:07:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:11 smithi132 ceph-mon[23807]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:11 smithi132 ceph-mon[23807]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3199996049"}]': finished 2024-05-09T04:07:11.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:11 smithi132 ceph-mon[23807]: osdmap e111: 8 total, 8 up, 8 in 2024-05-09T04:07:11.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:11 smithi078 ceph-mon[22905]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:11.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:11 smithi078 ceph-mon[22905]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3199996049"}]': finished 2024-05-09T04:07:11.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:11 smithi078 ceph-mon[22905]: osdmap e111: 8 total, 8 up, 8 in 2024-05-09T04:07:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:11 smithi078 ceph-mon[27629]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:11 smithi078 ceph-mon[27629]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/3199996049"}]': finished 2024-05-09T04:07:11.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:11 smithi078 ceph-mon[27629]: osdmap e111: 8 total, 8 up, 8 in 2024-05-09T04:07:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:13 smithi132 ceph-mon[23807]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-09T04:07:13.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:13 smithi078 ceph-mon[22905]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-09T04:07:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:13 smithi078 ceph-mon[27629]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-09T04:07:15.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:07:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:07:14] "GET /metrics HTTP/1.1" 200 38060 "" "Prometheus/2.33.4" 2024-05-09T04:07:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:15 smithi132 ceph-mon[23807]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-09T04:07:15.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:15 smithi078 ceph-mon[22905]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-09T04:07:15.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:15 smithi078 ceph-mon[27629]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-09T04:07:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:16 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:16 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:16.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:16 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:17.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:17 smithi132 ceph-mon[23807]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-09T04:07:17.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:17 smithi078 ceph-mon[22905]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-09T04:07:17.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:17 smithi078 ceph-mon[27629]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-09T04:07:19.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:19 smithi132 ceph-mon[23807]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-05-09T04:07:19.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:19 smithi078 ceph-mon[22905]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-05-09T04:07:19.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:19 smithi078 ceph-mon[27629]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-05-09T04:07:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:21 smithi078 ceph-mon[22905]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:07:21.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:21 smithi078 ceph-mon[27629]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:07:21.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:21 smithi132 ceph-mon[23807]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:07:23.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:23 smithi132 ceph-mon[23807]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 869 B/s rd, 0 op/s 2024-05-09T04:07:23.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:23 smithi078 ceph-mon[22905]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 869 B/s rd, 0 op/s 2024-05-09T04:07:23.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:23 smithi078 ceph-mon[27629]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 869 B/s rd, 0 op/s 2024-05-09T04:07:24.537 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:24 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:07:24.537 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:24 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:07:24.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:24 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:07:25.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:07:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:07:24] "GET /metrics HTTP/1.1" 200 38061 "" "Prometheus/2.33.4" 2024-05-09T04:07:25.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:25 smithi132 ceph-mon[23807]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:25 smithi078 ceph-mon[22905]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:25 smithi078 ceph-mon[27629]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:26 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:26.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:26 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:26 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:27.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:27 smithi132 ceph-mon[23807]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:27.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:27 smithi078 ceph-mon[22905]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:27.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:27 smithi078 ceph-mon[27629]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:29.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:29 smithi132 ceph-mon[23807]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:29.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:29 smithi078 ceph-mon[22905]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:29.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:29 smithi078 ceph-mon[27629]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:31 smithi132 ceph-mon[23807]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:31.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:31 smithi078 ceph-mon[22905]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:31.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:31 smithi078 ceph-mon[27629]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:33.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:33 smithi132 ceph-mon[23807]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:33.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:33 smithi078 ceph-mon[22905]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:33.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:33 smithi078 ceph-mon[27629]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:35.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:07:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:07:34] "GET /metrics HTTP/1.1" 200 38061 "" "Prometheus/2.33.4" 2024-05-09T04:07:35.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:35 smithi132 ceph-mon[23807]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:35.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:35 smithi078 ceph-mon[22905]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:35.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:35 smithi078 ceph-mon[27629]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:36.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:36 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:36.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:36 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:36.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:36 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:37 smithi132 ceph-mon[23807]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:37 smithi078 ceph-mon[22905]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:37.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:37 smithi078 ceph-mon[27629]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:39 smithi132 ceph-mon[23807]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:07:39.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:39 smithi078 ceph-mon[22905]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:39.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:07:39.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:39 smithi078 ceph-mon[27629]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:39.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:07:41.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:41 smithi132 ceph-mon[23807]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:41.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:41 smithi078 ceph-mon[22905]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:41.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:41 smithi078 ceph-mon[27629]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:43.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:43 smithi132 ceph-mon[23807]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:43.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:43 smithi078 ceph-mon[27629]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:43.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:43 smithi078 ceph-mon[22905]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:45.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:07:44 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:07:44] "GET /metrics HTTP/1.1" 200 38061 "" "Prometheus/2.33.4" 2024-05-09T04:07:45.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:45 smithi132 ceph-mon[23807]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:45.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:45 smithi078 ceph-mon[22905]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:45.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:45 smithi078 ceph-mon[27629]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:46.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:46 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:46.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:46 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:46.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:46 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:47.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:47 smithi132 ceph-mon[23807]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:47.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:47 smithi078 ceph-mon[22905]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:47.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:47 smithi078 ceph-mon[27629]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:49.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:49 smithi132 ceph-mon[23807]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:49.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:49 smithi078 ceph-mon[22905]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:49.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:49 smithi078 ceph-mon[27629]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:51.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:51 smithi132 ceph-mon[23807]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:51.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:51 smithi078 ceph-mon[22905]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:51.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:51 smithi078 ceph-mon[27629]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:53.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:53 smithi132 ceph-mon[23807]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:53.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:53 smithi078 ceph-mon[22905]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:53 smithi078 ceph-mon[27629]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:54.563 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:54 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:07:54.564 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:54 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:07:54.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:54 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:07:55.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:07:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:07:54] "GET /metrics HTTP/1.1" 200 38060 "" "Prometheus/2.33.4" 2024-05-09T04:07:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:55 smithi132 ceph-mon[23807]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:55.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:55 smithi078 ceph-mon[22905]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:55.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:55 smithi078 ceph-mon[27629]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:07:56.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:56 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:56.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:56 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:56.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:56 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:07:57.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:57 smithi132 ceph-mon[23807]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:57.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:57 smithi078 ceph-mon[22905]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:57.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:57 smithi078 ceph-mon[27629]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:07:59 smithi132 ceph-mon[23807]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:07:59 smithi078 ceph-mon[22905]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:07:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:07:59 smithi078 ceph-mon[27629]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:01 smithi132 ceph-mon[23807]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:01 smithi078 ceph-mon[22905]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:01.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:01 smithi078 ceph-mon[27629]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:03.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:03 smithi132 ceph-mon[23807]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:03.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:03 smithi078 ceph-mon[22905]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:03.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:03 smithi078 ceph-mon[27629]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:05.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:08:04 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:08:04] "GET /metrics HTTP/1.1" 200 38060 "" "Prometheus/2.33.4" 2024-05-09T04:08:05.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:05 smithi132 ceph-mon[23807]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:05.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:05 smithi078 ceph-mon[22905]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:05.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:05 smithi078 ceph-mon[27629]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:06.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:06 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:06.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:06 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:06.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:06 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:07.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:07 smithi132 ceph-mon[23807]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:07.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:07 smithi078 ceph-mon[27629]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:07.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:07 smithi078 ceph-mon[22905]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:09.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:09 smithi132 ceph-mon[23807]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:09.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:09 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:08:09.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:09 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:08:09.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:09 smithi078 ceph-mon[27629]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:09.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:09 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:08:09.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:09 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:08:09.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:09 smithi078 ceph-mon[22905]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:09.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:09 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:08:09.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:09 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:08:10.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:10 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:08:10.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:10 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:08:10.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:10 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:08:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:10 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:08:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:10 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:08:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:10 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:08:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:10 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:08:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:10 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:08:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:10 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:08:11.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:11 smithi132 ceph-mon[23807]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:11.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:11 smithi078 ceph-mon[22905]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:11.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:11 smithi078 ceph-mon[27629]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:13 smithi132 ceph-mon[23807]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:13.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:13 smithi078 ceph-mon[22905]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:13 smithi078 ceph-mon[27629]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:15.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:08:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:08:14] "GET /metrics HTTP/1.1" 200 38060 "" "Prometheus/2.33.4" 2024-05-09T04:08:15.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:15 smithi132 ceph-mon[23807]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:15.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:15 smithi078 ceph-mon[22905]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:15.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:15 smithi078 ceph-mon[27629]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:16 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:16 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:16.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:16 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:17.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:17 smithi132 ceph-mon[23807]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:17.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:17 smithi078 ceph-mon[27629]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:17.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:17 smithi078 ceph-mon[22905]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:19.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:19 smithi132 ceph-mon[23807]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:19.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:19 smithi078 ceph-mon[22905]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:19.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:19 smithi078 ceph-mon[27629]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:21.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:21 smithi132 ceph-mon[23807]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:21.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:21 smithi078 ceph-mon[22905]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:21.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:21 smithi078 ceph-mon[27629]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:23.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:23 smithi132 ceph-mon[23807]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:23.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:23 smithi078 ceph-mon[22905]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:23.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:23 smithi078 ceph-mon[27629]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:24.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:24 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:08:24.596 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:24 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:08:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:24 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:08:25.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:08:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:08:24] "GET /metrics HTTP/1.1" 200 38058 "" "Prometheus/2.33.4" 2024-05-09T04:08:25.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:25 smithi132 ceph-mon[23807]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:25.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:25 smithi078 ceph-mon[22905]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:25 smithi078 ceph-mon[27629]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:26.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:26 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:26 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:26 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:27.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:27 smithi132 ceph-mon[23807]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:27.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:27 smithi078 ceph-mon[22905]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:27.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:27 smithi078 ceph-mon[27629]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:29.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:29 smithi132 ceph-mon[23807]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:29.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:29 smithi078 ceph-mon[22905]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:29.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:29 smithi078 ceph-mon[27629]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:31 smithi132 ceph-mon[23807]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:31.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:31 smithi078 ceph-mon[22905]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:31.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:31 smithi078 ceph-mon[27629]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:33.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:33 smithi132 ceph-mon[23807]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:33.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:33 smithi078 ceph-mon[22905]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:33.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:33 smithi078 ceph-mon[27629]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:35.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:08:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:08:34] "GET /metrics HTTP/1.1" 200 38058 "" "Prometheus/2.33.4" 2024-05-09T04:08:35.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:35 smithi132 ceph-mon[23807]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:35.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:35 smithi078 ceph-mon[22905]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:35.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:35 smithi078 ceph-mon[27629]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:36.671 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:36 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:36.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:36 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:36.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:36 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:37.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:37 smithi132 ceph-mon[23807]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:37.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:37 smithi078 ceph-mon[22905]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:37.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:37 smithi078 ceph-mon[27629]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:39 smithi132 ceph-mon[23807]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:08:39.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:39 smithi078 ceph-mon[27629]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:39.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:08:39.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:39 smithi078 ceph-mon[22905]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:39.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:08:41.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:41 smithi132 ceph-mon[23807]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:41.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:41 smithi078 ceph-mon[22905]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:41.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:41 smithi078 ceph-mon[27629]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:43.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:43 smithi132 ceph-mon[23807]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:43.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:43 smithi078 ceph-mon[22905]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:43.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:43 smithi078 ceph-mon[27629]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:45.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:08:44 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:08:44] "GET /metrics HTTP/1.1" 200 38061 "" "Prometheus/2.33.4" 2024-05-09T04:08:45.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:45 smithi132 ceph-mon[23807]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:45.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:45 smithi078 ceph-mon[27629]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:45.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:45 smithi078 ceph-mon[22905]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:46.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:46 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:46.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:46 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:46.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:46 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:47.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:47 smithi132 ceph-mon[23807]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:47.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:47 smithi078 ceph-mon[22905]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:47.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:47 smithi078 ceph-mon[27629]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:49.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:49 smithi132 ceph-mon[23807]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:49.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:49 smithi078 ceph-mon[22905]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:49.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:49 smithi078 ceph-mon[27629]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:51.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:51 smithi132 ceph-mon[23807]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:51.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:51 smithi078 ceph-mon[22905]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:51.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:51 smithi078 ceph-mon[27629]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:53.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:53 smithi132 ceph-mon[23807]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:53.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:53 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:08:53.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:53 smithi078 ceph-mon[22905]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:53.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:53 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:08:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:53 smithi078 ceph-mon[27629]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:53 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:08:55.253 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:08:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:08:54] "GET /metrics HTTP/1.1" 200 38069 "" "Prometheus/2.33.4" 2024-05-09T04:08:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:55 smithi132 ceph-mon[23807]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:55 smithi078 ceph-mon[22905]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:55.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:55 smithi078 ceph-mon[27629]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:08:56.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:56 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:56.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:56 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:56.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:56 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:08:57.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:57 smithi132 ceph-mon[23807]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:57.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:57 smithi078 ceph-mon[22905]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:57.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:57 smithi078 ceph-mon[27629]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:59.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:08:59 smithi132 ceph-mon[23807]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:08:59 smithi078 ceph-mon[22905]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:08:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:08:59 smithi078 ceph-mon[27629]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:01 smithi132 ceph-mon[23807]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:01.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:01 smithi078 ceph-mon[22905]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:01.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:01 smithi078 ceph-mon[27629]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:03.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:03 smithi132 ceph-mon[23807]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:03.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:03 smithi078 ceph-mon[22905]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:03.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:03 smithi078 ceph-mon[27629]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:05.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:09:04 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:09:04] "GET /metrics HTTP/1.1" 200 38069 "" "Prometheus/2.33.4" 2024-05-09T04:09:05.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:05 smithi132 ceph-mon[23807]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:05.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:05 smithi078 ceph-mon[22905]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:05.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:05 smithi078 ceph-mon[27629]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:06.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:06 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:06.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:06 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:06.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:06 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:07.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:07 smithi132 ceph-mon[23807]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:07.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:07 smithi078 ceph-mon[22905]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:07.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:07 smithi078 ceph-mon[27629]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:09.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:09 smithi132 ceph-mon[23807]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:09.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:09 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:09.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:09 smithi078 ceph-mon[27629]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:09.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:09 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:09.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:09 smithi078 ceph-mon[22905]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:09.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:09 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:10.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:10 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:09:10.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:10 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:09:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:10 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:09:11.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:11 smithi132 ceph-mon[23807]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:11.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:11 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:09:11.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:11 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:09:11.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:11 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:09:11.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:11 smithi078 ceph-mon[22905]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:11.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:11 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:09:11.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:11 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:09:11.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:11 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:09:11.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:11 smithi078 ceph-mon[27629]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:11.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:11 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:09:11.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:11 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:09:11.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:11 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:09:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:13 smithi132 ceph-mon[23807]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:13.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:13 smithi078 ceph-mon[22905]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:13.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:13 smithi078 ceph-mon[27629]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:15.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:09:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:09:14] "GET /metrics HTTP/1.1" 200 38066 "" "Prometheus/2.33.4" 2024-05-09T04:09:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:15 smithi132 ceph-mon[23807]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:15.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:15 smithi078 ceph-mon[22905]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:15.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:15 smithi078 ceph-mon[27629]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:16.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:16 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:16 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:16 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:17.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:17 smithi132 ceph-mon[23807]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:17.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:17 smithi078 ceph-mon[22905]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:17.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:17 smithi078 ceph-mon[27629]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:19.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:19 smithi132 ceph-mon[23807]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:19.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:19 smithi078 ceph-mon[22905]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:19.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:19 smithi078 ceph-mon[27629]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:21.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:21 smithi132 ceph-mon[23807]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:21.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:21 smithi078 ceph-mon[22905]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:21.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:21 smithi078 ceph-mon[27629]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:23.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:23 smithi132 ceph-mon[23807]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:23.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:23 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:23.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:23 smithi078 ceph-mon[22905]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:23.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:23 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:23.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:23 smithi078 ceph-mon[27629]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:23.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:23 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:25.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:09:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:09:24] "GET /metrics HTTP/1.1" 200 38071 "" "Prometheus/2.33.4" 2024-05-09T04:09:25.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:25 smithi132 ceph-mon[23807]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:25.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:25 smithi078 ceph-mon[27629]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:25 smithi078 ceph-mon[22905]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:26.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:26 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:26.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:26 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:26.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:26 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:27.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:27 smithi132 ceph-mon[23807]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:27.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:27 smithi078 ceph-mon[22905]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:27.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:27 smithi078 ceph-mon[27629]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:29.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:29 smithi132 ceph-mon[23807]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:29.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:29 smithi078 ceph-mon[22905]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:29.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:29 smithi078 ceph-mon[27629]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:31 smithi132 ceph-mon[23807]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:31.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:31 smithi078 ceph-mon[22905]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:31.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:31 smithi078 ceph-mon[27629]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:33.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:33 smithi132 ceph-mon[23807]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:33.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:33 smithi078 ceph-mon[22905]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:33.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:33 smithi078 ceph-mon[27629]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:35.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:09:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:09:34] "GET /metrics HTTP/1.1" 200 38071 "" "Prometheus/2.33.4" 2024-05-09T04:09:35.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:35 smithi132 ceph-mon[23807]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:35.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:35 smithi078 ceph-mon[27629]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:35.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:35 smithi078 ceph-mon[22905]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:36.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:36 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:36.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:36 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:36.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:36 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:37.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:37 smithi132 ceph-mon[23807]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:37.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:37 smithi078 ceph-mon[22905]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:37.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:37 smithi078 ceph-mon[27629]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:39.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:39 smithi132 ceph-mon[23807]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:39.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:39 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:39.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:39 smithi078 ceph-mon[27629]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:39.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:39 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:39.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:39 smithi078 ceph-mon[22905]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:39.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:39 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:41.588 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:41 smithi078 ceph-mon[22905]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:41.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:41 smithi078 ceph-mon[27629]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:41.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:41 smithi132 ceph-mon[23807]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:41.975 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T04:09:43.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:43 smithi078 ceph-mon[22905]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:43.559 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:43 smithi078 ceph-mon[27629]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:43.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:43 smithi132 ceph-mon[23807]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:44.006 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:09:44.006 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (12m) 2m ago 20m 23.4M - 0.23.0 ba2b418f427c f44cb50f5f35 2024-05-09T04:09:44.006 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (12m) 2m ago 20m 54.2M - 8.3.5 dad864ee21e9 33b2a450dbe1 2024-05-09T04:09:44.006 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (2m) 2m ago 19m 57.7M - 3.5 e1d6a67b021e 33a9b5cd3d16 2024-05-09T04:09:44.006 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283 running (15m) 2m ago 24m 509M - 19.0.0-3631-gc66c553a af4a4df59e0a 4a3f6038cf25 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (9m) 2m ago 26m 462M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (26m) 2m ago 26m 70.5M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (25m) 2m ago 25m 55.9M 2048M 17.2.0 e1d6a67b021e a71bae52559d 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (25m) 2m ago 25m 55.5M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (12m) 2m ago 21m 17.9M - 1.3.1 1dbe0e931976 4a3cc1414851 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (12m) 2m ago 21m 16.9M - 1.3.1 1dbe0e931976 666630a53114 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (24m) 2m ago 24m 53.1M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (24m) 2m ago 24m 56.9M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (23m) 2m ago 23m 50.3M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (23m) 2m ago 23m 53.3M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (23m) 2m ago 23m 55.0M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (22m) 2m ago 22m 52.2M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (22m) 2m ago 22m 51.3M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:09:44.007 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (21m) 2m ago 21m 55.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:09:44.008 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (2m) 2m ago 21m 41.9M - 2.33.4 514e6a882f6e fef66365cfcd 2024-05-09T04:09:44.008 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (19m) 2m ago 19m 76.7M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:09:44.008 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (19m) 2m ago 19m 78.0M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:09:44.657 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ls' 2024-05-09T04:09:45.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:09:44 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:09:44] "GET /metrics HTTP/1.1" 200 38072 "" "Prometheus/2.33.4" 2024-05-09T04:09:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:45 smithi078 ceph-mon[22905]: from='client.25369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:09:45.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:45 smithi078 ceph-mon[22905]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:45 smithi078 ceph-mon[27629]: from='client.25369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:09:45.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:45 smithi078 ceph-mon[27629]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:45.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:45 smithi132 ceph-mon[23807]: from='client.25369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:09:45.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:45 smithi132 ceph-mon[23807]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:46.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:46 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:46 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:46.667 INFO:teuthology.orchestra.run.smithi078.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-05-09T04:09:46.667 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager ?:9093,9094 1/1 2m ago 21m smithi078=a;count:1 2024-05-09T04:09:46.667 INFO:teuthology.orchestra.run.smithi078.stdout:grafana ?:3000 1/1 2m ago 21m smithi132=a;count:1 2024-05-09T04:09:46.667 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo ?:5000 1/1 2m ago 19m count:1 2024-05-09T04:09:46.667 INFO:teuthology.orchestra.run.smithi078.stdout:mgr 2/2 2m ago 25m smithi078=y;smithi132=x;count:2 2024-05-09T04:09:46.667 INFO:teuthology.orchestra.run.smithi078.stdout:mon 3/3 2m ago 25m smithi078:172.21.15.78=a;smithi078:[v2:172.21.15.78:3301,v1:172.21.15.78:6790]=c;smithi132:172.21.15.132=b;count:3 2024-05-09T04:09:46.667 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter ?:9100 2/2 2m ago 21m smithi078=a;smithi132=b;count:2 2024-05-09T04:09:46.668 INFO:teuthology.orchestra.run.smithi078.stdout:osd 8 2m ago - 2024-05-09T04:09:46.668 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus ?:9095 1/1 2m ago 21m smithi132=a;count:1 2024-05-09T04:09:46.668 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo ?:8000 2/2 2m ago 19m count:2 2024-05-09T04:09:46.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:46 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:47.327 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions' 2024-05-09T04:09:47.633 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:47 smithi078 ceph-mon[22905]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:47.633 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:47 smithi078 ceph-mon[27629]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:47.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:47 smithi132 ceph-mon[23807]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:48.657 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:48 smithi078 ceph-mon[22905]: from='client.15474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:09:48.657 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:48 smithi078 ceph-mon[27629]: from='client.15474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:09:48.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:48 smithi132 ceph-mon[23807]: from='client.15474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:09:49.368 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:09:49.368 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:09:49.368 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-09T04:09:49.368 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:09:49.368 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:09:49.368 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:09:49.368 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:09:49.368 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:09:49.368 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T04:09:49.369 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:09:49.369 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": {}, 2024-05-09T04:09:49.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:09:49.369 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:09:49.369 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:09:49.369 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:09:49.369 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-09T04:09:49.369 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:09:49.369 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:09:49.369 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:09:49.657 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:49 smithi078 ceph-mon[27629]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:49.658 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:49 smithi078 ceph-mon[22905]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:49.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:49 smithi132 ceph-mon[23807]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:49.913 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-05-09T04:09:50.468 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:50 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2649156557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:09:50.468 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:50 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2649156557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:09:50.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:50 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2649156557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:09:51.495 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:51 smithi078 ceph-mon[27629]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:51.495 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:51 smithi078 ceph-mon[22905]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:51.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:51 smithi132 ceph-mon[23807]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:53.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:53 smithi078 ceph-mon[22905]: from='client.15486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:09:53.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:53 smithi078 ceph-mon[22905]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:53.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:53 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:53.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:53 smithi078 ceph-mon[27629]: from='client.15486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:09:53.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:53 smithi078 ceph-mon[27629]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:53.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:53 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:53.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:53 smithi132 ceph-mon[23807]: from='client.15486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:09:53.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:53 smithi132 ceph-mon[23807]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:53.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:53 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:54.239 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:09:54.838 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-09T04:09:55.242 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:09:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ::ffff:172.21.15.132 - - [09/May/2024:04:09:54] "GET /metrics HTTP/1.1" 200 38064 "" "Prometheus/2.33.4" 2024-05-09T04:09:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:55 smithi132 ceph-mon[23807]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:09:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:55 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:09:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:55 smithi132 ceph-mon[23807]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:55 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:09:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:55 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:09:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:55 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:09:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:55 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:09:55.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:55 smithi132 ceph-mon[23807]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:09:55.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[27629]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:09:55.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:09:55.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[27629]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:55.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:09:55.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:09:55.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:09:55.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:09:55.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[27629]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:09:55.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[22905]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:09:55.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:09:55.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[22905]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:09:55.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:09:55.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:09:55.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:09:55.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:09:55.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:55 smithi078 ceph-mon[22905]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:09:56.496 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:56 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:56.497 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:56 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:56.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:56 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:09:57.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:57 smithi132 ceph-mon[23807]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:57.506 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:57 smithi078 ceph-mon[27629]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:57.507 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:57 smithi078 ceph-mon[22905]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:09:58.292 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:09:59.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:09:58 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T04:09:58.596+0000 7f6fed282700 -1 mgr handle_mgr_map I was active but no longer am 2024-05-09T04:09:59.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:09:58 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ignoring --setuser ceph since I am not root 2024-05-09T04:09:59.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:09:58 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: ignoring --setgroup ceph since I am not root 2024-05-09T04:09:59.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mgr[44012]: -- 172.21.15.132:0/3207488783 <== mon.2 v2:172.21.15.132:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562cfb4f85a0 con 0x562cfb4ccc00 2024-05-09T04:09:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:09:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mon[23807]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:09:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mon[23807]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:09:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:09:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mon[23807]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-09T04:09:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mon[23807]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-09T04:09:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mon[23807]: Failing over to other MGR 2024-05-09T04:09:59.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mon[23807]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-09T04:09:59.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-09T04:09:59.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mon[23807]: osdmap e112: 8 total, 8 up, 8 in 2024-05-09T04:09:59.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mon[23807]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:09:59.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:58 smithi132 ceph-mon[23807]: from='client.25393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:09:59.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:09:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[27629]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:09:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[27629]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:09:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:09:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[27629]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-09T04:09:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[27629]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-09T04:09:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[27629]: Failing over to other MGR 2024-05-09T04:09:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[27629]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-09T04:09:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-09T04:09:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[27629]: osdmap e112: 8 total, 8 up, 8 in 2024-05-09T04:09:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[27629]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:09:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[27629]: from='client.25393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:09:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:09:58] ENGINE Bus STOPPING 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:09:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:09:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:09:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:09:58] ENGINE Bus STOPPED 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:09:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:09:58] ENGINE Bus STARTING 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:09:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:09:58] ENGINE Serving on http://:::9283 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:09:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:09:58] ENGINE Bus STARTED 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[22905]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[22905]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[22905]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[22905]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-09T04:09:59.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[22905]: Failing over to other MGR 2024-05-09T04:09:59.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[22905]: from='mgr.24959 172.21.15.132:0/2838340701' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-09T04:09:59.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-09T04:09:59.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[22905]: osdmap e112: 8 total, 8 up, 8 in 2024-05-09T04:09:59.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[22905]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:09:59.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:58 smithi078 ceph-mon[22905]: from='client.25393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:09:59.918 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.24959 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-05-09T04:09:59.918 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: mgrmap e37: y(active, starting, since 0.999286s) 2024-05-09T04:09:59.918 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:09:59.918 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:09:59.918 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:09:59.918 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T04:09:59.918 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: Manager daemon y is now available 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:59.919 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:09:59.920 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T04:09:59.920 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[27629]: [09/May/2024:04:09:59] ENGINE Bus STARTING 2024-05-09T04:09:59.920 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.24959 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-05-09T04:09:59.920 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: mgrmap e37: y(active, starting, since 0.999286s) 2024-05-09T04:09:59.920 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:09:59.920 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:09:59.920 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:09:59.920 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T04:09:59.920 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:09:59.920 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:09:59.920 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:09:59.920 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: Manager daemon y is now available 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T04:09:59.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:09:59 smithi078 ceph-mon[22905]: [09/May/2024:04:09:59] ENGINE Bus STARTING 2024-05-09T04:09:59.957 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:09:59.957 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (12m) 2m ago 20m 23.4M - 0.23.0 ba2b418f427c f44cb50f5f35 2024-05-09T04:09:59.957 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (12m) 3m ago 20m 54.2M - 8.3.5 dad864ee21e9 33b2a450dbe1 2024-05-09T04:09:59.957 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (2m) 2m ago 19m 57.7M - 3.5 e1d6a67b021e 33a9b5cd3d16 2024-05-09T04:09:59.957 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283 running (15m) 3m ago 25m 509M - 19.0.0-3631-gc66c553a af4a4df59e0a 4a3f6038cf25 2024-05-09T04:09:59.957 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (9m) 2m ago 26m 462M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:09:59.957 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (26m) 2m ago 27m 70.5M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T04:09:59.957 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (25m) 3m ago 25m 55.9M 2048M 17.2.0 e1d6a67b021e a71bae52559d 2024-05-09T04:09:59.957 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (25m) 2m ago 25m 55.5M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T04:09:59.957 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (12m) 2m ago 21m 17.9M - 1.3.1 1dbe0e931976 4a3cc1414851 2024-05-09T04:09:59.957 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (12m) 3m ago 21m 16.9M - 1.3.1 1dbe0e931976 666630a53114 2024-05-09T04:09:59.957 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (24m) 2m ago 24m 53.1M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:09:59.958 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (24m) 2m ago 24m 56.9M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:09:59.958 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (24m) 2m ago 24m 50.3M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:09:59.958 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (23m) 2m ago 23m 53.3M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:09:59.958 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (23m) 3m ago 23m 55.0M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:09:59.958 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (22m) 3m ago 22m 52.2M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:09:59.958 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (22m) 3m ago 22m 51.3M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:09:59.958 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (22m) 3m ago 22m 55.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:09:59.958 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (3m) 3m ago 21m 41.9M - 2.33.4 514e6a882f6e fef66365cfcd 2024-05-09T04:09:59.958 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (19m) 2m ago 19m 76.7M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:09:59.958 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (19m) 3m ago 19m 78.0M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:10:00.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.24959 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-05-09T04:10:00.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: mgrmap e37: y(active, starting, since 0.999286s) 2024-05-09T04:10:00.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:10:00.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:10:00.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:10:00.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: Manager daemon y is now available 2024-05-09T04:10:00.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:00.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:10:00.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:10:00.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T04:10:00.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:09:59 smithi132 ceph-mon[23807]: [09/May/2024:04:09:59] ENGINE Bus STARTING 2024-05-09T04:10:00.386 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:10:00.387 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:10:00.387 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-09T04:10:00.387 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:10:00.387 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:10:00.387 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 1 2024-05-09T04:10:00.387 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:10:00.387 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:10:00.387 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T04:10:00.387 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:10:00.387 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": {}, 2024-05-09T04:10:00.387 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:10:00.387 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:10:00.388 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:10:00.388 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:10:00.388 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-09T04:10:00.388 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 1 2024-05-09T04:10:00.388 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:10:00.388 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:10:00.641 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[27629]: [09/May/2024:04:09:59] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T04:10:00.641 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[27629]: [09/May/2024:04:09:59] ENGINE Serving on http://172.21.15.78:8765 2024-05-09T04:10:00.641 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[27629]: [09/May/2024:04:09:59] ENGINE Bus STARTED 2024-05-09T04:10:00.641 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[27629]: mgrmap e38: y(active, since 2s) 2024-05-09T04:10:00.641 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[27629]: from='client.15498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:00.641 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[27629]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:10:00.641 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[27629]: from='client.15528 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:00.642 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[27629]: overall HEALTH_OK 2024-05-09T04:10:00.642 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/687582663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:00.642 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[22905]: [09/May/2024:04:09:59] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T04:10:00.642 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[22905]: [09/May/2024:04:09:59] ENGINE Serving on http://172.21.15.78:8765 2024-05-09T04:10:00.642 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[22905]: [09/May/2024:04:09:59] ENGINE Bus STARTED 2024-05-09T04:10:00.642 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[22905]: mgrmap e38: y(active, since 2s) 2024-05-09T04:10:00.642 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[22905]: from='client.15498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:00.642 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[22905]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:10:00.642 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[22905]: from='client.15528 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:00.642 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[22905]: overall HEALTH_OK 2024-05-09T04:10:00.642 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:00 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/687582663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:00.743 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:10:00.743 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:10:00.743 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:10:00.743 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-09T04:10:00.743 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-05-09T04:10:00.743 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-05-09T04:10:00.743 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-05-09T04:10:00.743 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/2 daemons upgraded", 2024-05-09T04:10:00.743 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-05-09T04:10:00.743 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:10:00.743 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:10:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:00 smithi132 ceph-mon[23807]: [09/May/2024:04:09:59] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T04:10:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:00 smithi132 ceph-mon[23807]: [09/May/2024:04:09:59] ENGINE Serving on http://172.21.15.78:8765 2024-05-09T04:10:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:00 smithi132 ceph-mon[23807]: [09/May/2024:04:09:59] ENGINE Bus STARTED 2024-05-09T04:10:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:00 smithi132 ceph-mon[23807]: mgrmap e38: y(active, since 2s) 2024-05-09T04:10:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:00 smithi132 ceph-mon[23807]: from='client.15498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:00 smithi132 ceph-mon[23807]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:10:01.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:00 smithi132 ceph-mon[23807]: from='client.15528 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:01.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:00 smithi132 ceph-mon[23807]: overall HEALTH_OK 2024-05-09T04:10:01.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:00 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/687582663' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:01 smithi132 ceph-mon[23807]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:10:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:01 smithi132 ceph-mon[23807]: mgrmap e39: y(active, since 3s) 2024-05-09T04:10:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:01 smithi132 ceph-mon[23807]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:01 smithi078 ceph-mon[22905]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:10:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:01 smithi078 ceph-mon[22905]: mgrmap e39: y(active, since 3s) 2024-05-09T04:10:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:01 smithi078 ceph-mon[22905]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:01.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:01 smithi078 ceph-mon[27629]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:10:01.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:01 smithi078 ceph-mon[27629]: mgrmap e39: y(active, since 3s) 2024-05-09T04:10:01.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:01 smithi078 ceph-mon[27629]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:04.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:03 smithi132 ceph-mon[23807]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:10:04.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:03 smithi132 ceph-mon[23807]: mgrmap e40: y(active, since 5s) 2024-05-09T04:10:04.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:03 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:04.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:03 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:04.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:03 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:04.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:03 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:04.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:03 smithi078 ceph-mon[22905]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:10:04.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:03 smithi078 ceph-mon[22905]: mgrmap e40: y(active, since 5s) 2024-05-09T04:10:04.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:03 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:04.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:03 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:04.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:03 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:04.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:03 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:04.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:03 smithi078 ceph-mon[27629]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:10:04.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:03 smithi078 ceph-mon[27629]: mgrmap e40: y(active, since 5s) 2024-05-09T04:10:04.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:03 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:04.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:03 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:04.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:03 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:04.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:03 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:10:06.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:06.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:10:06.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T04:10:06.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T04:10:06.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:06.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:10:06.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:10:06.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:10:06.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:05 smithi132 ceph-mon[23807]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:10:06.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:10:06.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:06.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:06.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:10:06.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T04:10:06.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T04:10:06.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: from='client.25318 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:06.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:10:06.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:10:06.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:10:06.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:05 smithi078 ceph-mon[22905]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:10:06.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:04:10:06] ENGINE Bus STARTING 2024-05-09T04:10:06.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: CherryPy Checker: 2024-05-09T04:10:06.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: The Application mounted at '' has an empty config. 2024-05-09T04:10:06.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: 2024-05-09T04:10:06.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:04:10:06] ENGINE Serving on http://:::9283 2024-05-09T04:10:06.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[44008]: [09/May/2024:04:10:06] ENGINE Bus STARTED 2024-05-09T04:10:07.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:10:07.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:10:07.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:10:07.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:10:07.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:07.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:10:07.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: Standby manager daemon x started 2024-05-09T04:10:07.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/2156098229' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T04:10:07.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/2156098229' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:10:07.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/2156098229' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T04:10:07.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:06 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/2156098229' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:10:07.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:10:07.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:10:07.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:10:07.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:10:07.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:07.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:10:07.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: Standby manager daemon x started 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/2156098229' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/2156098229' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/2156098229' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/2156098229' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: Reconfiguring iscsi.foo.smithi078.vnplzc (dependencies changed)... 2024-05-09T04:10:07.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:10:07.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:07.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: Reconfiguring daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:10:07.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: Standby manager daemon x started 2024-05-09T04:10:07.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/2156098229' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T04:10:07.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/2156098229' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:10:07.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/2156098229' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T04:10:07.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:06 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/2156098229' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:10:07.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:07 smithi078 ceph-mon[22905]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T04:10:07.966 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:07 smithi078 ceph-mon[22905]: mgrmap e41: y(active, since 9s), standbys: x 2024-05-09T04:10:07.966 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:07 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T04:10:07.966 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:07 smithi078 ceph-mon[27629]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T04:10:07.966 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:07 smithi078 ceph-mon[27629]: mgrmap e41: y(active, since 9s), standbys: x 2024-05-09T04:10:07.966 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:07 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T04:10:07.971 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:07 smithi132 ceph-mon[23807]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T04:10:07.971 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:07 smithi132 ceph-mon[23807]: mgrmap e41: y(active, since 9s), standbys: x 2024-05-09T04:10:07.971 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:07 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T04:10:10.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:10 smithi132 ceph-mon[23807]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T04:10:10.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:10 smithi078 ceph-mon[22905]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T04:10:10.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:10:10 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:10:10.315+0000 7fd1285d9700 -1 log_channel(cephadm) log [ERR] : Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:10:10.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:10 smithi078 ceph-mon[27629]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T04:10:11.422 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:11 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:11.422 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:11 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:11.422 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:11 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3195129497' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:10:11.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:11 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:11.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:11 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:11.515 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:11 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3195129497' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:10:11.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:11 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:11.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:11 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:11.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:11 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3195129497' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:10:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:12 smithi132 ceph-mon[23807]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T04:10:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:12 smithi132 ceph-mon[23807]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:10:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:12 smithi132 ceph-mon[23807]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T04:10:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:12 smithi132 ceph-mon[23807]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 0 B/s wr, 72 op/s 2024-05-09T04:10:12.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:12 smithi078 ceph-mon[22905]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T04:10:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:12 smithi078 ceph-mon[22905]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:10:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:12 smithi078 ceph-mon[22905]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T04:10:12.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:12 smithi078 ceph-mon[22905]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 0 B/s wr, 72 op/s 2024-05-09T04:10:12.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:12 smithi078 ceph-mon[27629]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-09T04:10:12.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:12 smithi078 ceph-mon[27629]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:10:12.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:12 smithi078 ceph-mon[27629]: Reconfiguring daemon prometheus.a on smithi132 2024-05-09T04:10:12.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:12 smithi078 ceph-mon[27629]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 0 B/s wr, 72 op/s 2024-05-09T04:10:14.254 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:13 smithi132 systemd[1]: Stopping Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:10:14.254 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:13 smithi132 bash[58620]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus.a" found: no such container 2024-05-09T04:10:14.254 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:10:14.003Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-09T04:10:14.254 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:10:14.004Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-09T04:10:14.254 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:10:14.004Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-09T04:10:14.254 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:10:14.004Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-09T04:10:14.254 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:10:14.004Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-09T04:10:14.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:10:14.004Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-09T04:10:14.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:10:14.004Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-09T04:10:14.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:10:14.004Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-09T04:10:14.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:10:14.004Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-09T04:10:14.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:10:14.004Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-09T04:10:14.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:10:14.004Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-09T04:10:14.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[55781]: ts=2024-05-09T04:10:14.004Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-09T04:10:14.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 podman[58631]: 2024-05-09 04:10:14.014282398 +0000 UTC m=+0.136057741 container died fef66365cfcd89d72ab19e17961f97006461318230af92408cb3ba925086fa62 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:10:14.621 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:14 smithi132 ceph-mon[23807]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 66 op/s 2024-05-09T04:10:14.621 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:14 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:10:14.621 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 podman[58631]: 2024-05-09 04:10:14.394925392 +0000 UTC m=+0.516700790 container cleanup fef66365cfcd89d72ab19e17961f97006461318230af92408cb3ba925086fa62 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:10:14.621 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 bash[58631]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a 2024-05-09T04:10:14.621 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 bash[58657]: Error: no container with name or ID "ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus.a" found: no such container 2024-05-09T04:10:14.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:14 smithi078 ceph-mon[22905]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 66 op/s 2024-05-09T04:10:14.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:14 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:10:14.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:14 smithi078 ceph-mon[27629]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 66 op/s 2024-05-09T04:10:14.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:14 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:10:14.872 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 podman[58645]: 2024-05-09 04:10:14.621136573 +0000 UTC m=+0.604072170 container remove fef66365cfcd89d72ab19e17961f97006461318230af92408cb3ba925086fa62 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:10:14.872 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-09T04:10:14.872 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 systemd[1]: Stopped Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:10:14.872 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service: Consumed 1.408s CPU time. 2024-05-09T04:10:14.872 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 systemd[1]: Starting Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:10:14.872 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 podman[58726]: 2024-05-09 04:10:14.757012234 +0000 UTC m=+0.019537631 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-09T04:10:14.872 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:14 smithi132 podman[58726]: 2024-05-09 04:10:14.872128514 +0000 UTC m=+0.134653906 container create 307643c1dad33f8199768a4c7c749c0591c7e1a36fca7b5476063f7451d6666e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:10:15.141 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 podman[58726]: 2024-05-09 04:10:15.065842795 +0000 UTC m=+0.328368184 container init 307643c1dad33f8199768a4c7c749c0591c7e1a36fca7b5476063f7451d6666e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:10:15.141 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 podman[58726]: 2024-05-09 04:10:15.068621683 +0000 UTC m=+0.331147075 container start 307643c1dad33f8199768a4c7c749c0591c7e1a36fca7b5476063f7451d6666e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:10:15.141 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.091Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-09T04:10:15.141 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.091Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-09T04:10:15.141 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.091Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-09T04:10:15.141 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.092Z caller=main.go:518 level=info host_details="(Linux 5.14.0-444.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024 x86_64 smithi132 (none))" 2024-05-09T04:10:15.141 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.092Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-09T04:10:15.141 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.092Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-09T04:10:15.142 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.093Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-09T04:10:15.142 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.094Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-09T04:10:15.142 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.094Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-09T04:10:15.142 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.097Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-09T04:10:15.142 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.098Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=843.131µs 2024-05-09T04:10:15.142 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.098Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-09T04:10:15.142 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 bash[58726]: 307643c1dad33f8199768a4c7c749c0591c7e1a36fca7b5476063f7451d6666e 2024-05-09T04:10:15.503 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 systemd[1]: Started Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:10:15.503 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.313Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-05-09T04:10:15.875 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.561Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-05-09T04:10:15.876 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.764Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-05-09T04:10:15.876 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.771Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-05-09T04:10:15.876 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.773Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-05-09T04:10:15.876 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.774Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-05-09T04:10:15.876 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.774Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=49.269µs wal_replay_duration=675.738215ms total_replay_duration=676.648461ms 2024-05-09T04:10:15.876 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.778Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-09T04:10:15.876 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.778Z caller=main.go:947 level=info msg="TSDB started" 2024-05-09T04:10:15.876 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.778Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-09T04:10:15.876 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.798Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.108976ms db_storage=805ns remote_storage=1.6µs web_handler=310ns query_engine=690ns scrape=370.345µs scrape_sd=112.51µs notify=14.888µs notify_sd=16.788µs rules=19.222827ms 2024-05-09T04:10:15.876 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:10:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:10:15.798Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-09T04:10:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 66 op/s 2024-05-09T04:10:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:10:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:10:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: Adding iSCSI gateway http://:@172.21.15.78:5000 to Dashboard 2024-05-09T04:10:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:10:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:10:16.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T04:10:16.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T04:10:16.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:16.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:10:16.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:10:16.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 66 op/s 2024-05-09T04:10:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:10:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:10:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: Adding iSCSI gateway http://:@172.21.15.78:5000 to Dashboard 2024-05-09T04:10:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:10:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:10:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T04:10:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T04:10:16.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 66 op/s 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: Adding iSCSI gateway http://:@172.21.15.78:5000 to Dashboard 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T04:10:16.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi078"}]: dispatch 2024-05-09T04:10:16.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:16.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:10:16.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:10:16.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:17.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:17 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:10:17.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:17 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T04:10:17.489 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:17 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:17.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:17 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:10:17.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:17 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T04:10:17.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:17 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:17.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:17 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:10:17.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:17 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T04:10:17.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:17 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:18.371 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:18 smithi132 ceph-mon[23807]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 0 B/s wr, 126 op/s 2024-05-09T04:10:18.371 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:18 smithi132 ceph-mon[23807]: Upgrade: Updating mgr.x 2024-05-09T04:10:18.372 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:18 smithi132 ceph-mon[23807]: Deploying daemon mgr.x on smithi132 2024-05-09T04:10:18.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:18 smithi078 ceph-mon[22905]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 0 B/s wr, 126 op/s 2024-05-09T04:10:18.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:18 smithi078 ceph-mon[22905]: Upgrade: Updating mgr.x 2024-05-09T04:10:18.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:18 smithi078 ceph-mon[22905]: Deploying daemon mgr.x on smithi132 2024-05-09T04:10:18.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:18 smithi078 ceph-mon[27629]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 0 B/s wr, 126 op/s 2024-05-09T04:10:18.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:18 smithi078 ceph-mon[27629]: Upgrade: Updating mgr.x 2024-05-09T04:10:18.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:18 smithi078 ceph-mon[27629]: Deploying daemon mgr.x on smithi132 2024-05-09T04:10:18.940 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:18 smithi132 systemd[1]: Stopping Ceph mgr.x for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:10:19.209 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:18 smithi132 podman[59071]: 2024-05-09 04:10:18.93994299 +0000 UTC m=+0.149096221 container died 4a3f6038cf259a266e375524efe780b91398de440f8bb466cc29814be955b51e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=) 2024-05-09T04:10:19.503 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:19 smithi132 podman[59071]: 2024-05-09 04:10:19.32558464 +0000 UTC m=+0.534737909 container cleanup 4a3f6038cf259a266e375524efe780b91398de440f8bb466cc29814be955b51e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-05-09T04:10:19.504 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:19 smithi132 bash[59071]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x 2024-05-09T04:10:19.940 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:19 smithi132 podman[59085]: 2024-05-09 04:10:19.618455551 +0000 UTC m=+0.678077546 container remove 4a3f6038cf259a266e375524efe780b91398de440f8bb466cc29814be955b51e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, ceph=True) 2024-05-09T04:10:19.940 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:19 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-05-09T04:10:19.940 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:19 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-05-09T04:10:19.940 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:19 smithi132 systemd[1]: Stopped Ceph mgr.x for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:10:19.940 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:19 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.x.service: Consumed 1min 698ms CPU time. 2024-05-09T04:10:20.213 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:19 smithi132 systemd[1]: Starting Ceph mgr.x for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:10:20.213 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:20 smithi132 podman[59217]: 2024-05-09 04:10:20.060352534 +0000 UTC m=+0.018325004 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:10:20.213 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:20 smithi132 podman[59217]: 2024-05-09 04:10:20.161810698 +0000 UTC m=+0.119783198 container create 0569600c45d0e8db14297e5248fe44486118ac690e3477a3cc142f663286d8cc (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-05-09T04:10:20.472 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:20 smithi132 ceph-mon[23807]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:10:20.472 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:20 smithi132 podman[59217]: 2024-05-09 04:10:20.397833257 +0000 UTC m=+0.355805719 container init 0569600c45d0e8db14297e5248fe44486118ac690e3477a3cc142f663286d8cc (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240507) 2024-05-09T04:10:20.473 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:20 smithi132 podman[59217]: 2024-05-09 04:10:20.400578038 +0000 UTC m=+0.358550509 container start 0569600c45d0e8db14297e5248fe44486118ac690e3477a3cc142f663286d8cc (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:10:20.473 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:20 smithi132 ceph-mgr[59236]: -- 172.21.15.132:0/4056128436 <== mon.2 v2:172.21.15.132:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a8d4e325a0 con 0x55a8d4e06800 2024-05-09T04:10:20.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:20 smithi078 ceph-mon[22905]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:10:20.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:20 smithi078 ceph-mon[27629]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:10:20.753 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:20 smithi132 bash[59217]: 0569600c45d0e8db14297e5248fe44486118ac690e3477a3cc142f663286d8cc 2024-05-09T04:10:20.753 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:20 smithi132 systemd[1]: Started Ceph mgr.x for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:10:21.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:21 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:21.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:21 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:21.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:21 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:21.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:21 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:21.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:21 smithi132 ceph-mon[23807]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:10:22.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:21 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:22.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:21 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:22.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:21 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:22.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:21 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:22.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:21 smithi078 ceph-mon[22905]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:10:22.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:21 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:22.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:21 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:22.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:21 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:22.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:21 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:22.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:21 smithi078 ceph-mon[27629]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:10:24.498 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:24 smithi132 ceph-mon[23807]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:10:24.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:24 smithi078 ceph-mon[22905]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:10:24.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:24 smithi078 ceph-mon[27629]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:10:26.235 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:25 smithi132 ceph-mon[23807]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:10:26.236 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:25 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:26.236 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:25 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:26.236 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:25 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:26.236 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:25 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:26.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:25 smithi078 ceph-mon[22905]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:10:26.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:25 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:26.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:25 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:26.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:25 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:26.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:25 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:25 smithi078 ceph-mon[27629]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:10:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:25 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:25 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:25 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:25 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:27.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:26 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:27.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:26 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:27.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:26 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:27.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:26 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:27.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:26 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:27.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:26 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:28.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:28 smithi132 ceph-mon[23807]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:10:28.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:28 smithi132 ceph-mon[23807]: Standby manager daemon x restarted 2024-05-09T04:10:28.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:28 smithi132 ceph-mon[23807]: Standby manager daemon x started 2024-05-09T04:10:28.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:28 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/1510483377' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T04:10:28.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:28 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/1510483377' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:10:28.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:28 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/1510483377' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T04:10:28.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:28 smithi132 ceph-mon[23807]: from='mgr.? 172.21.15.132:0/1510483377' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:10:28.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: [09/May/2024:04:10:27] ENGINE Bus STARTING 2024-05-09T04:10:28.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: CherryPy Checker: 2024-05-09T04:10:28.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: The Application mounted at '' has an empty config. 2024-05-09T04:10:28.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: 2024-05-09T04:10:28.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: [09/May/2024:04:10:28] ENGINE Serving on http://:::9283 2024-05-09T04:10:28.257 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:10:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: [09/May/2024:04:10:28] ENGINE Bus STARTED 2024-05-09T04:10:28.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[22905]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:10:28.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[22905]: Standby manager daemon x restarted 2024-05-09T04:10:28.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[22905]: Standby manager daemon x started 2024-05-09T04:10:28.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/1510483377' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T04:10:28.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/1510483377' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:10:28.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/1510483377' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T04:10:28.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[22905]: from='mgr.? 172.21.15.132:0/1510483377' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:10:28.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[27629]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-09T04:10:28.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[27629]: Standby manager daemon x restarted 2024-05-09T04:10:28.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[27629]: Standby manager daemon x started 2024-05-09T04:10:28.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/1510483377' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T04:10:28.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/1510483377' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:10:28.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/1510483377' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T04:10:28.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:28 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/1510483377' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:10:29.214 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:29 smithi132 ceph-mon[23807]: mgrmap e42: y(active, since 30s), standbys: x 2024-05-09T04:10:29.214 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:29 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:10:29.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:29 smithi078 ceph-mon[22905]: mgrmap e42: y(active, since 30s), standbys: x 2024-05-09T04:10:29.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:29 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:10:29.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:29 smithi078 ceph-mon[27629]: mgrmap e42: y(active, since 30s), standbys: x 2024-05-09T04:10:29.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:29 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:10:29.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:10:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:10:29] "GET /metrics HTTP/1.1" 200 38070 "" "Prometheus/2.33.4" 2024-05-09T04:10:30.342 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:30.342 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.342 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.342 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:30.343 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:10:30.343 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.343 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:30.343 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.343 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.343 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-09T04:10:30.345 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-05-09T04:10:30.345 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-09T04:10:30.345 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-05-09T04:10:30.345 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.345 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.345 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.345 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.345 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.345 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.345 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.345 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.346 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.346 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.346 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.346 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.346 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.346 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:30.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:30.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:10:30.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:30.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-09T04:10:30.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-05-09T04:10:30.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:30 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:10:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:30.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:30.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:30 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:31.106 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:10:31.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all mgr 2024-05-09T04:10:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all crash 2024-05-09T04:10:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all mds 2024-05-09T04:10:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:10:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-09T04:10:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:10:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all nfs 2024-05-09T04:10:31.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:10:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all mgr 2024-05-09T04:10:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all crash 2024-05-09T04:10:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all mds 2024-05-09T04:10:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:10:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-09T04:10:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:10:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all nfs 2024-05-09T04:10:31.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:31 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:10:31.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:31 smithi132 ceph-mon[23807]: Upgrade: Setting container_image for all mgr 2024-05-09T04:10:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:31 smithi132 ceph-mon[23807]: Upgrade: Setting container_image for all crash 2024-05-09T04:10:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:31 smithi132 ceph-mon[23807]: Upgrade: Setting container_image for all mds 2024-05-09T04:10:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:31 smithi132 ceph-mon[23807]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:10:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:31 smithi132 ceph-mon[23807]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-09T04:10:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:31 smithi132 ceph-mon[23807]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:10:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:31 smithi132 ceph-mon[23807]: Upgrade: Setting container_image for all nfs 2024-05-09T04:10:31.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:31 smithi132 ceph-mon[23807]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:10:31.860 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (13m) 6s ago 21m 23.1M - 0.23.0 ba2b418f427c f44cb50f5f35 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (13m) 7s ago 20m 53.5M - 8.3.5 dad864ee21e9 33b2a450dbe1 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (22s) 6s ago 20m 50.4M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (11s) 7s ago 25m 62.0M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (10m) 6s ago 27m 509M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (27m) 6s ago 27m 76.2M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (25m) 7s ago 25m 60.7M 2048M 17.2.0 e1d6a67b021e a71bae52559d 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (26m) 6s ago 26m 60.5M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (13m) 6s ago 21m 17.2M - 1.3.1 1dbe0e931976 4a3cc1414851 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (13m) 7s ago 21m 18.3M - 1.3.1 1dbe0e931976 666630a53114 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (25m) 6s ago 25m 54.5M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (25m) 6s ago 25m 58.5M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (24m) 6s ago 24m 50.7M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:10:31.861 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (24m) 6s ago 24m 53.9M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:10:31.862 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (23m) 7s ago 23m 57.7M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:10:31.862 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (23m) 7s ago 23m 52.4M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:10:31.862 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (23m) 7s ago 23m 53.2M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:10:31.862 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (22m) 7s ago 22m 57.4M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:10:31.862 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (17s) 7s ago 22m 51.0M - 2.33.4 514e6a882f6e 307643c1dad3 2024-05-09T04:10:31.862 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (20m) 6s ago 20m 77.7M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:10:31.862 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (20m) 7s ago 20m 78.8M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:10:32.011 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:31 smithi078 systemd[1]: Stopping Ceph node-exporter.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:10:32.309 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:10:32.310 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:10:32.310 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-09T04:10:32.310 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:10:32.310 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:10:32.310 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:10:32.310 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:10:32.310 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:10:32.310 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T04:10:32.310 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:10:32.310 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": {}, 2024-05-09T04:10:32.310 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:10:32.311 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:10:32.311 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:10:32.311 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:10:32.311 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-09T04:10:32.311 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:10:32.311 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:10:32.311 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:10:32.464 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:32 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:32.465 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:32 smithi078 ceph-mon[27629]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:32.465 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:32 smithi078 ceph-mon[27629]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:32.465 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:32 smithi078 ceph-mon[27629]: Upgrade: Updating node-exporter.a (1/2) 2024-05-09T04:10:32.465 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:32 smithi078 ceph-mon[27629]: Deploying daemon node-exporter.a on smithi078 2024-05-09T04:10:32.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:32 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:32.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:32 smithi078 ceph-mon[22905]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:32.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:32 smithi078 ceph-mon[22905]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:32.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:32 smithi078 ceph-mon[22905]: Upgrade: Updating node-exporter.a (1/2) 2024-05-09T04:10:32.465 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:32 smithi078 ceph-mon[22905]: Deploying daemon node-exporter.a on smithi078 2024-05-09T04:10:32.466 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:32 smithi078 podman[76369]: 2024-05-09 04:10:32.190097944 +0000 UTC m=+0.106991403 container died 4a3cc1414851bdd6a1171351bed2e3d0d00f54d6c0b55571568cf9dc60f02a7f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T04:10:32.669 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:10:32.669 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:10:32.670 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:10:32.670 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-09T04:10:32.670 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-05-09T04:10:32.670 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-05-09T04:10:32.670 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-05-09T04:10:32.670 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/2 daemons upgraded", 2024-05-09T04:10:32.670 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading node-exporter daemons", 2024-05-09T04:10:32.670 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:10:32.670 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:10:32.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:32 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:32.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:32 smithi132 ceph-mon[23807]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:32.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:32 smithi132 ceph-mon[23807]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:32.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:32 smithi132 ceph-mon[23807]: Upgrade: Updating node-exporter.a (1/2) 2024-05-09T04:10:32.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:32 smithi132 ceph-mon[23807]: Deploying daemon node-exporter.a on smithi078 2024-05-09T04:10:32.761 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:32 smithi078 podman[76369]: 2024-05-09 04:10:32.571353422 +0000 UTC m=+0.488246881 container cleanup 4a3cc1414851bdd6a1171351bed2e3d0d00f54d6c0b55571568cf9dc60f02a7f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T04:10:32.761 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:32 smithi078 bash[76369]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a 2024-05-09T04:10:33.108 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:32 smithi078 podman[76383]: 2024-05-09 04:10:32.797621641 +0000 UTC m=+0.611453069 container remove 4a3cc1414851bdd6a1171351bed2e3d0d00f54d6c0b55571568cf9dc60f02a7f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T04:10:33.108 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:32 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-05-09T04:10:33.108 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:32 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-05-09T04:10:33.108 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:32 smithi078 systemd[1]: Stopped Ceph node-exporter.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:10:33.108 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:32 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.a.service: Consumed 5.259s CPU time. 2024-05-09T04:10:33.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:33 smithi078 ceph-mon[22905]: from='client.15588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:33 smithi078 ceph-mon[22905]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:33.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:33 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2335677616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:33.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:33 smithi078 ceph-mon[27629]: from='client.15588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:33.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:33 smithi078 ceph-mon[27629]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:33.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:33 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2335677616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:33.512 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:33 smithi078 systemd[1]: Starting Ceph node-exporter.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:10:33.512 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:33 smithi078 bash[76541]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-05-09T04:10:33.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:33 smithi132 ceph-mon[23807]: from='client.15588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:33.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:33 smithi132 ceph-mon[23807]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:33.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:33 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2335677616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:34.011 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:33 smithi078 bash[76541]: Getting image source signatures 2024-05-09T04:10:34.012 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:33 smithi078 bash[76541]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-05-09T04:10:34.012 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:33 smithi078 bash[76541]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-05-09T04:10:34.012 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:33 smithi078 bash[76541]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-05-09T04:10:34.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:34 smithi132 ceph-mon[23807]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:34.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:34 smithi132 ceph-mon[23807]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:34.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:34 smithi078 ceph-mon[22905]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:34.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:34 smithi078 ceph-mon[22905]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:34.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:34 smithi078 ceph-mon[27629]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:34.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:34 smithi078 ceph-mon[27629]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:10:35.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-mon[22905]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:35.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:35 smithi078 ceph-mon[27629]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:35.512 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 bash[76541]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-05-09T04:10:35.512 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 bash[76541]: Writing manifest to image destination 2024-05-09T04:10:35.655 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:35 smithi132 ceph-mon[23807]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:35.835 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 podman[76541]: 2024-05-09 04:10:35.501719191 +0000 UTC m=+2.293926051 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-05-09T04:10:35.835 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 podman[76541]: 2024-05-09 04:10:35.625145914 +0000 UTC m=+2.417352773 container create c0004282e7ce84fe0f2052628381fcac473b42c0f4d37c197764d40ddc2089ec (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T04:10:35.835 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 podman[76541]: 2024-05-09 04:10:35.835483001 +0000 UTC m=+2.627689860 container init c0004282e7ce84fe0f2052628381fcac473b42c0f4d37c197764d40ddc2089ec (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T04:10:36.173 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 podman[76541]: 2024-05-09 04:10:35.83898285 +0000 UTC m=+2.631189714 container start c0004282e7ce84fe0f2052628381fcac473b42c0f4d37c197764d40ddc2089ec (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T04:10:36.173 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.843Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-05-09T04:10:36.173 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.843Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-05-09T04:10:36.173 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.843Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-05-09T04:10:36.173 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.843Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-05-09T04:10:36.173 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-05-09T04:10:36.173 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-09T04:10:36.173 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-05-09T04:10:36.173 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=arp 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=bcache 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=bonding 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=btrfs 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=conntrack 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=cpu 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=diskstats 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=dmi 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=edac 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=entropy 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=filefd 2024-05-09T04:10:36.174 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=filesystem 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=hwmon 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=infiniband 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=ipvs 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=loadavg 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=mdadm 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=meminfo 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=netclass 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=netdev 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=netstat 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=nfs 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=nfsd 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=nvme 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=os 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=pressure 2024-05-09T04:10:36.175 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=rapl 2024-05-09T04:10:36.176 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=schedstat 2024-05-09T04:10:36.176 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=selinux 2024-05-09T04:10:36.176 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=sockstat 2024-05-09T04:10:36.176 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=softnet 2024-05-09T04:10:36.176 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=stat 2024-05-09T04:10:36.176 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=tapestats 2024-05-09T04:10:36.176 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=textfile 2024-05-09T04:10:36.176 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-05-09T04:10:36.176 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=time 2024-05-09T04:10:36.176 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-05-09T04:10:36.177 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=uname 2024-05-09T04:10:36.177 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=vmstat 2024-05-09T04:10:36.177 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=xfs 2024-05-09T04:10:36.177 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=node_exporter.go:117 level=info collector=zfs 2024-05-09T04:10:36.177 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-05-09T04:10:36.177 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a[76591]: ts=2024-05-09T04:10:35.844Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-05-09T04:10:36.177 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 bash[76541]: c0004282e7ce84fe0f2052628381fcac473b42c0f4d37c197764d40ddc2089ec 2024-05-09T04:10:36.177 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:10:35 smithi078 systemd[1]: Started Ceph node-exporter.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:10:37.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:36 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:37.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:36 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:37.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:36 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:37.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:36 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:37.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:36 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:37.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:36 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:38.253 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:38 smithi132 systemd[1]: Stopping Ceph node-exporter.b for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:10:38.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:37 smithi132 ceph-mon[23807]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:38.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:37 smithi078 ceph-mon[22905]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:38.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:37 smithi078 ceph-mon[27629]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:38.753 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:38 smithi132 podman[60635]: 2024-05-09 04:10:38.494172714 +0000 UTC m=+0.121554214 container died 666630a5311453012e0ff03c4836df7fc38870fcde3d09d8822e79ffefc6608c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T04:10:39.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:38 smithi132 podman[60635]: 2024-05-09 04:10:38.866229112 +0000 UTC m=+0.493610615 container cleanup 666630a5311453012e0ff03c4836df7fc38870fcde3d09d8822e79ffefc6608c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T04:10:39.142 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:38 smithi132 bash[60635]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b 2024-05-09T04:10:39.486 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:39 smithi132 podman[60650]: 2024-05-09 04:10:39.142511807 +0000 UTC m=+0.652239775 container remove 666630a5311453012e0ff03c4836df7fc38870fcde3d09d8822e79ffefc6608c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T04:10:39.486 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:39 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-05-09T04:10:39.486 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:39 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-05-09T04:10:39.486 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:39 smithi132 systemd[1]: Stopped Ceph node-exporter.b for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:10:39.487 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:39 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.b.service: Consumed 5.379s CPU time. 2024-05-09T04:10:39.487 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:39 smithi132 ceph-mon[23807]: Upgrade: Updating node-exporter.b (2/2) 2024-05-09T04:10:39.487 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:39 smithi132 ceph-mon[23807]: Deploying daemon node-exporter.b on smithi132 2024-05-09T04:10:39.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:39 smithi078 ceph-mon[22905]: Upgrade: Updating node-exporter.b (2/2) 2024-05-09T04:10:39.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:39 smithi078 ceph-mon[22905]: Deploying daemon node-exporter.b on smithi132 2024-05-09T04:10:39.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:39 smithi078 ceph-mon[27629]: Upgrade: Updating node-exporter.b (2/2) 2024-05-09T04:10:39.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:39 smithi078 ceph-mon[27629]: Deploying daemon node-exporter.b on smithi132 2024-05-09T04:10:39.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:10:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:10:39] "GET /metrics HTTP/1.1" 200 38070 "" "Prometheus/2.33.4" 2024-05-09T04:10:39.753 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:39 smithi132 systemd[1]: Starting Ceph node-exporter.b for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:10:39.753 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:39 smithi132 bash[60783]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-05-09T04:10:40.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:40 smithi132 ceph-mon[23807]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:40.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:40 smithi132 bash[60783]: Getting image source signatures 2024-05-09T04:10:40.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:40 smithi132 bash[60783]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-05-09T04:10:40.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:40 smithi132 bash[60783]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-05-09T04:10:40.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:40 smithi132 bash[60783]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-05-09T04:10:40.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:40 smithi078 ceph-mon[27629]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:40.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:40 smithi078 ceph-mon[22905]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:42.503 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:42 smithi132 bash[60783]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-05-09T04:10:42.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:42 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:42.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:42 smithi132 ceph-mon[23807]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:42.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:42 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:42.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:42 smithi078 ceph-mon[27629]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:42 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:42 smithi078 ceph-mon[22905]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:42.912 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:42 smithi132 bash[60783]: Writing manifest to image destination 2024-05-09T04:10:43.196 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:42 smithi132 podman[60783]: 2024-05-09 04:10:42.845639423 +0000 UTC m=+3.259621875 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-05-09T04:10:43.196 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:42 smithi132 podman[60783]: 2024-05-09 04:10:42.99467722 +0000 UTC m=+3.408659667 container create 71aa9559f36fb97f3325669dd846883aefe83d02d775bf83100b09d0db5fddb9 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T04:10:43.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 podman[60783]: 2024-05-09 04:10:43.196865509 +0000 UTC m=+3.610847969 container init 71aa9559f36fb97f3325669dd846883aefe83d02d775bf83100b09d0db5fddb9 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T04:10:43.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 podman[60783]: 2024-05-09 04:10:43.200014882 +0000 UTC m=+3.613997331 container start 71aa9559f36fb97f3325669dd846883aefe83d02d775bf83100b09d0db5fddb9 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T04:10:43.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.204Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-05-09T04:10:43.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.204Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-05-09T04:10:43.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.204Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-05-09T04:10:43.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-09T04:10:43.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-05-09T04:10:43.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-05-09T04:10:43.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-05-09T04:10:43.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=arp 2024-05-09T04:10:43.504 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=bcache 2024-05-09T04:10:43.505 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=bonding 2024-05-09T04:10:43.505 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=btrfs 2024-05-09T04:10:43.505 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=conntrack 2024-05-09T04:10:43.505 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=cpu 2024-05-09T04:10:43.505 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-05-09T04:10:43.505 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=diskstats 2024-05-09T04:10:43.505 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=dmi 2024-05-09T04:10:43.505 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=edac 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=entropy 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=filefd 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=filesystem 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=hwmon 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=infiniband 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=ipvs 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=loadavg 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=mdadm 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=meminfo 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=netclass 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=netdev 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=netstat 2024-05-09T04:10:43.506 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=nfs 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=nfsd 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=nvme 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=os 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=pressure 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=rapl 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=schedstat 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=selinux 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=sockstat 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=softnet 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=stat 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=tapestats 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=textfile 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-05-09T04:10:43.507 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=time 2024-05-09T04:10:43.508 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-05-09T04:10:43.508 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=uname 2024-05-09T04:10:43.508 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=vmstat 2024-05-09T04:10:43.508 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=xfs 2024-05-09T04:10:43.508 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.205Z caller=node_exporter.go:117 level=info collector=zfs 2024-05-09T04:10:43.508 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.206Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-05-09T04:10:43.508 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b[60833]: ts=2024-05-09T04:10:43.206Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-05-09T04:10:43.508 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 bash[60783]: 71aa9559f36fb97f3325669dd846883aefe83d02d775bf83100b09d0db5fddb9 2024-05-09T04:10:43.508 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:10:43 smithi132 systemd[1]: Started Ceph node-exporter.b for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:10:44.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:44 smithi132 ceph-mon[23807]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:44 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:44 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:44 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:44.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:44 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:10:44.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:44 smithi078 ceph-mon[22905]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:44.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:44 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:44.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:44 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:44.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:44 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:44.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:44 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:10:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:44 smithi078 ceph-mon[27629]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:44 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:44 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:44 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:44 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:10:46.502 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:46 smithi132 ceph-mon[23807]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:46.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:46 smithi078 ceph-mon[22905]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:46 smithi078 ceph-mon[27629]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:48.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:48 smithi132 ceph-mon[23807]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:48 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:48 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:48 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:48.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:48 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:48.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:48 smithi078 ceph-mon[27629]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:48.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:48 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:48.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:48 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:48.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:48 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:48.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:48 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:48 smithi078 ceph-mon[22905]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:48 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:48 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:48 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:48.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:48 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:49.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:10:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:10:49] "GET /metrics HTTP/1.1" 200 38065 "" "Prometheus/2.33.4" 2024-05-09T04:10:49.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:49 smithi078 ceph-mon[22905]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:49.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:49 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:49.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:49 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:49.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:49 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:49.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:49 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:49.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:49 smithi078 ceph-mon[27629]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:49.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:49 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:49.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:49 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:49.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:49 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:49.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:49 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:50.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:49 smithi132 ceph-mon[23807]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:50.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:49 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:50.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:49 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:50.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:49 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:50.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:49 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:52.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:51 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:52.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:51 smithi078 ceph-mon[22905]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:52.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:51 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:52.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:51 smithi078 ceph-mon[27629]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:52.150 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:51 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:10:52.151 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:51 smithi132 ceph-mon[23807]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:10:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:53 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:10:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:53.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:53.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:53 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:10:54.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:10:54 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:10:54.333+0000 7fd1285d9700 -1 log_channel(cephadm) log [ERR] : Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:10:56.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:55 smithi132 ceph-mon[23807]: Upgrade: Updating prometheus.a 2024-05-09T04:10:56.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:55 smithi132 ceph-mon[23807]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:10:56.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:55 smithi132 ceph-mon[23807]: Deploying daemon prometheus.a on smithi132 2024-05-09T04:10:56.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:55 smithi132 ceph-mon[23807]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:56.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:55 smithi078 ceph-mon[22905]: Upgrade: Updating prometheus.a 2024-05-09T04:10:56.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:55 smithi078 ceph-mon[22905]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:10:56.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:55 smithi078 ceph-mon[22905]: Deploying daemon prometheus.a on smithi132 2024-05-09T04:10:56.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:55 smithi078 ceph-mon[22905]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:56.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:55 smithi078 ceph-mon[27629]: Upgrade: Updating prometheus.a 2024-05-09T04:10:56.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:55 smithi078 ceph-mon[27629]: Invalid JSON format for multi-cluster config: Expecting value: line 1 column 1 (char 0) 2024-05-09T04:10:56.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:55 smithi078 ceph-mon[27629]: Deploying daemon prometheus.a on smithi132 2024-05-09T04:10:56.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:55 smithi078 ceph-mon[27629]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:10:58.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:57 smithi132 ceph-mon[23807]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:58.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:57 smithi078 ceph-mon[22905]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:58.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:57 smithi078 ceph-mon[27629]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:10:59.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:10:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:10:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:10:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:10:59.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:10:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:10:59] "GET /metrics HTTP/1.1" 200 38066 "" "Prometheus/2.33.4" 2024-05-09T04:10:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:10:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:00.475 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:00 smithi078 ceph-mon[22905]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:00.476 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:00 smithi078 ceph-mon[27629]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:00.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:00 smithi132 ceph-mon[23807]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:02.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:02 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:02.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:02 smithi132 ceph-mon[23807]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:02.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:02 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:02.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:02 smithi078 ceph-mon[22905]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:02 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:02 smithi078 ceph-mon[27629]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:03.043 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:11:03.715 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (13m) 15s ago 21m 23.1M - 0.23.0 ba2b418f427c f44cb50f5f35 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (13m) 15s ago 21m 54.5M - 8.3.5 dad864ee21e9 33b2a450dbe1 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (53s) 15s ago 20m 50.5M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (43s) 15s ago 26m 464M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (10m) 15s ago 27m 515M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (28m) 15s ago 28m 76.5M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (26m) 15s ago 26m 61.1M 2048M 17.2.0 e1d6a67b021e a71bae52559d 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (26m) 15s ago 26m 60.9M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (28s) 15s ago 22m 7868k - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (20s) 15s ago 22m 3640k - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (25m) 15s ago 25m 54.7M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (25m) 15s ago 25m 58.7M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:11:03.716 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (25m) 15s ago 25m 50.9M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:11:03.717 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (24m) 15s ago 24m 54.1M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:11:03.717 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (24m) 15s ago 24m 57.7M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:11:03.717 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (24m) 15s ago 24m 52.6M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:11:03.717 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (23m) 15s ago 23m 53.4M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:11:03.717 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (23m) 15s ago 23m 57.5M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:11:03.717 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (48s) 15s ago 22m 62.0M - 2.33.4 514e6a882f6e 307643c1dad3 2024-05-09T04:11:03.717 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (20m) 15s ago 20m 77.9M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:11:03.717 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (20m) 15s ago 20m 79.2M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:11:04.130 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:11:04.130 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:11:04.130 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-09T04:11:04.130 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:11:04.130 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:11:04.131 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:11:04.131 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:11:04.131 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:11:04.131 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T04:11:04.131 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:11:04.131 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": {}, 2024-05-09T04:11:04.131 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:11:04.131 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:11:04.131 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:11:04.131 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:11:04.131 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-09T04:11:04.132 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:11:04.132 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:11:04.132 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:11:04.474 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:11:04.474 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:11:04.475 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:11:04.475 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-09T04:11:04.475 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-05-09T04:11:04.475 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-05-09T04:11:04.475 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-05-09T04:11:04.475 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/2 daemons upgraded", 2024-05-09T04:11:04.475 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading prometheus daemons", 2024-05-09T04:11:04.475 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:11:04.475 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:11:04.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:04 smithi078 ceph-mon[27629]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:04.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:04 smithi078 ceph-mon[27629]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:04.476 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:04 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3351185594' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:04.476 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:04 smithi078 ceph-mon[22905]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:04.476 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:04 smithi078 ceph-mon[22905]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:04.476 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:04 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3351185594' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:04.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:04 smithi132 ceph-mon[23807]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:04.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:04 smithi132 ceph-mon[23807]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:04.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:04 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/3351185594' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:05.335 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 systemd[1]: Stopping Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:11:05.335 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:05 smithi132 ceph-mon[23807]: from='client.25519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:05.335 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:05 smithi132 ceph-mon[23807]: from='client.25525 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:05.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:05 smithi078 ceph-mon[22905]: from='client.25519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:05.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:05 smithi078 ceph-mon[22905]: from='client.25525 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:05.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:05 smithi078 ceph-mon[27629]: from='client.25519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:05.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:05 smithi078 ceph-mon[27629]: from='client.25525 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:05.671 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:11:05.373Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-09T04:11:05.671 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:11:05.373Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-09T04:11:05.671 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:11:05.373Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-09T04:11:05.671 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:11:05.373Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-09T04:11:05.671 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:11:05.373Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-09T04:11:05.671 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:11:05.373Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-09T04:11:05.671 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:11:05.373Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-09T04:11:05.671 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:11:05.374Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-09T04:11:05.672 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:11:05.374Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-09T04:11:05.672 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:11:05.374Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-09T04:11:05.672 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:11:05.374Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-09T04:11:05.672 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[58741]: ts=2024-05-09T04:11:05.374Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-09T04:11:05.672 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 podman[62340]: 2024-05-09 04:11:05.383826835 +0000 UTC m=+0.131955648 container died 307643c1dad33f8199768a4c7c749c0591c7e1a36fca7b5476063f7451d6666e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:11:06.003 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 podman[62340]: 2024-05-09 04:11:05.777575309 +0000 UTC m=+0.525704130 container cleanup 307643c1dad33f8199768a4c7c749c0591c7e1a36fca7b5476063f7451d6666e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:11:06.003 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:05 smithi132 bash[62340]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a 2024-05-09T04:11:06.297 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 podman[62352]: 2024-05-09 04:11:06.003768718 +0000 UTC m=+0.622112850 container remove 307643c1dad33f8199768a4c7c749c0591c7e1a36fca7b5476063f7451d6666e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:11:06.297 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-09T04:11:06.297 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 systemd[1]: Stopped Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:11:06.297 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service: Consumed 1.322s CPU time. 2024-05-09T04:11:06.297 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:06 smithi132 ceph-mon[23807]: from='client.25537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:06.298 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:06 smithi132 ceph-mon[23807]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:06.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:06 smithi078 ceph-mon[22905]: from='client.25537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:06.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:06 smithi078 ceph-mon[22905]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:06.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:06 smithi078 ceph-mon[27629]: from='client.25537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:06 smithi078 ceph-mon[27629]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:06.564 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 systemd[1]: Starting Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:11:06.815 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 podman[62495]: 2024-05-09 04:11:06.468343887 +0000 UTC m=+0.019094416 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-05-09T04:11:06.815 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 podman[62495]: 2024-05-09 04:11:06.591943861 +0000 UTC m=+0.142694411 container create c02ff059e70485a9cb9db9cd016302432392369689621c6e354a4b1d9917d692 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:11:06.815 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 podman[62495]: 2024-05-09 04:11:06.815757333 +0000 UTC m=+0.366507852 container init c02ff059e70485a9cb9db9cd016302432392369689621c6e354a4b1d9917d692 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:11:07.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 podman[62495]: 2024-05-09 04:11:06.818653468 +0000 UTC m=+0.369403990 container start c02ff059e70485a9cb9db9cd016302432392369689621c6e354a4b1d9917d692 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:11:07.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.842Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-05-09T04:11:07.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.842Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-05-09T04:11:07.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.842Z caller=main.go:570 level=info host_details="(Linux 5.14.0-444.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Apr 26 18:25:52 UTC 2024 x86_64 smithi132 (none))" 2024-05-09T04:11:07.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.842Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-09T04:11:07.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.842Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-09T04:11:07.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.844Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-05-09T04:11:07.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.844Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-05-09T04:11:07.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.845Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-05-09T04:11:07.255 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.845Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.847Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.848Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=718.82µs 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.848Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.856Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.869Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.879Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.882Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.884Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.885Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.885Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.885Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.517µs wal_replay_duration=37.126132ms wbl_replay_duration=125ns total_replay_duration=37.896586ms 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.889Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.889Z caller=main.go:1029 level=info msg="TSDB started" 2024-05-09T04:11:07.256 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.889Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-09T04:11:07.257 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 bash[62495]: c02ff059e70485a9cb9db9cd016302432392369689621c6e354a4b1d9917d692 2024-05-09T04:11:07.257 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 systemd[1]: Started Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:11:07.257 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.910Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.810165ms db_storage=808ns remote_storage=1.129µs web_handler=419ns query_engine=763ns scrape=526.59µs scrape_sd=87.035µs notify=18.468µs notify_sd=14.974µs rules=19.856322ms tracing=5.708µs 2024-05-09T04:11:07.257 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.910Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-05-09T04:11:07.257 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:11:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:11:06.910Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-05-09T04:11:08.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:07 smithi132 ceph-mon[23807]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:08.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:07 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:08.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:07 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:08.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:07 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:08.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:07 smithi078 ceph-mon[22905]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:08.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:07 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:08.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:07 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:08.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:07 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:08.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:07 smithi078 ceph-mon[27629]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:08.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:07 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:08.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:07 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:08.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:07 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:10.486 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:10 smithi078 ceph-mon[22905]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:10.487 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:10 smithi078 ceph-mon[27629]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:10.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:10 smithi132 ceph-mon[23807]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:12.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:12 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:12 smithi132 ceph-mon[23807]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:12 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:12 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:12.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:12 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:12.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:12 smithi078 ceph-mon[22905]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:12.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:12 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:12.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:12 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:12.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:12 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:12.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:12 smithi078 ceph-mon[27629]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:12.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:12 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:12.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:12 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:13.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:13 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:13 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:13.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:13 smithi132 ceph-mon[23807]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:13.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:13 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:13.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:13 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:13.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:13 smithi078 ceph-mon[27629]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:13.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:13 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:13.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:13 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:13.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:13 smithi078 ceph-mon[22905]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:14.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:14 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:14.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:14 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:14.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:14 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:15.749 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:15 smithi132 ceph-mon[23807]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:15.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:15 smithi078 ceph-mon[22905]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:15.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:15 smithi078 ceph-mon[27629]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:16.978 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:16.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:16.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:11:16.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:11:16.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:16.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:11:16.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:11:16.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:16.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:11:16.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:16.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:16.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:16 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:17.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:17.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:17.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:11:17.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:11:17.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:17.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:11:17.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-09T04:11:17.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:17.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:17.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:17.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:17.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:17.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:17.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:17.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:17.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:17.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:17.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:16 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:18.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:17 smithi078 ceph-mon[27629]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:18.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:17 smithi078 ceph-mon[22905]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:18.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:17 smithi132 ceph-mon[23807]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:19.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:19 smithi132 ceph-mon[23807]: Upgrade: Updating alertmanager.a 2024-05-09T04:11:19.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:19 smithi132 ceph-mon[23807]: Deploying daemon alertmanager.a on smithi078 2024-05-09T04:11:19.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:19 smithi078 ceph-mon[22905]: Upgrade: Updating alertmanager.a 2024-05-09T04:11:19.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:19 smithi078 ceph-mon[22905]: Deploying daemon alertmanager.a on smithi078 2024-05-09T04:11:19.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:11:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:11:19] "GET /metrics HTTP/1.1" 200 38064 "" "Prometheus/2.43.0" 2024-05-09T04:11:19.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:19 smithi078 ceph-mon[27629]: Upgrade: Updating alertmanager.a 2024-05-09T04:11:19.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:19 smithi078 ceph-mon[27629]: Deploying daemon alertmanager.a on smithi078 2024-05-09T04:11:20.496 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:20 smithi078 ceph-mon[22905]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:20.496 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:20 smithi078 ceph-mon[27629]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:20.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:20 smithi132 ceph-mon[23807]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:22 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:22 smithi132 ceph-mon[23807]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:22.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:22 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:22.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:22 smithi078 ceph-mon[22905]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:22.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:22 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:22.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:22 smithi078 ceph-mon[27629]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:24.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:24 smithi132 ceph-mon[23807]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:24.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:24 smithi078 ceph-mon[22905]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:24.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:24 smithi078 ceph-mon[27629]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:26.066 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:25 smithi078 systemd[1]: Stopping Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:11:26.375 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:26 smithi078 ceph-mon[27629]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:26.375 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:26 smithi078 ceph-mon[22905]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:26.375 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:26 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[57024]: level=info ts=2024-05-09T04:11:26.066Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-05-09T04:11:26.375 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:26 smithi078 podman[78173]: 2024-05-09 04:11:26.116944027 +0000 UTC m=+0.177646271 container died f44cb50f5f358f021ca163840b8438a1783603d2554dc0078f0a6647a78c25c6 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T04:11:26.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:26 smithi132 ceph-mon[23807]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:26.759 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:26 smithi078 podman[78173]: 2024-05-09 04:11:26.541020029 +0000 UTC m=+0.601722283 container cleanup f44cb50f5f358f021ca163840b8438a1783603d2554dc0078f0a6647a78c25c6 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T04:11:26.759 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:26 smithi078 bash[78173]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a 2024-05-09T04:11:27.011 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:26 smithi078 podman[78187]: 2024-05-09 04:11:26.759126735 +0000 UTC m=+0.639424105 container remove f44cb50f5f358f021ca163840b8438a1783603d2554dc0078f0a6647a78c25c6 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T04:11:27.011 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:26 smithi078 podman[78187]: 2024-05-09 04:11:26.934772181 +0000 UTC m=+0.815069553 volume remove a1684abc154aa6fab9d4672b25496da89fddb483be351c009e6b16ae17dc2af8 2024-05-09T04:11:27.511 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:27 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-09T04:11:27.511 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:27 smithi078 systemd[1]: Stopped Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:11:27.512 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:27 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@alertmanager.a.service: Consumed 1.518s CPU time. 2024-05-09T04:11:27.512 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:27 smithi078 systemd[1]: Starting Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:11:27.951 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:27 smithi078 podman[78317]: 2024-05-09 04:11:27.535860362 +0000 UTC m=+0.018665129 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-05-09T04:11:27.952 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:27 smithi078 podman[78317]: 2024-05-09 04:11:27.644197348 +0000 UTC m=+0.127002115 volume create 2ac6bf7960f2b23a9d7e4e49a421a6b60f6fd4ddce90a1073e47070c29cd6f2c 2024-05-09T04:11:27.952 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:27 smithi078 podman[78317]: 2024-05-09 04:11:27.736133581 +0000 UTC m=+0.218938348 container create 3cd8b265bc1ddaf746671324335ed79ceb5f7ade0f2d3f772d90d79582f3bcb8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T04:11:28.240 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:28 smithi078 podman[78317]: 2024-05-09 04:11:28.064229048 +0000 UTC m=+0.547033815 container init 3cd8b265bc1ddaf746671324335ed79ceb5f7ade0f2d3f772d90d79582f3bcb8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T04:11:28.240 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:28 smithi078 podman[78317]: 2024-05-09 04:11:28.067191208 +0000 UTC m=+0.549995984 container start 3cd8b265bc1ddaf746671324335ed79ceb5f7ade0f2d3f772d90d79582f3bcb8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T04:11:28.240 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:28 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:11:28.080Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-05-09T04:11:28.240 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:28 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:11:28.080Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-05-09T04:11:28.240 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:28 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:11:28.081Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.78 port=9094 2024-05-09T04:11:28.240 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:28 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:11:28.082Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-09T04:11:28.240 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:28 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:11:28.106Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-09T04:11:28.240 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:28 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:11:28.107Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-09T04:11:28.240 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:28 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:11:28.109Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-05-09T04:11:28.240 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:28 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:11:28.109Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-05-09T04:11:28.241 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:28 smithi078 bash[78317]: 3cd8b265bc1ddaf746671324335ed79ceb5f7ade0f2d3f772d90d79582f3bcb8 2024-05-09T04:11:28.241 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:28 smithi078 systemd[1]: Started Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:11:28.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:28 smithi132 ceph-mon[23807]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:28.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:28 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:28 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:28 smithi078 ceph-mon[22905]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:28 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:28 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:28.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:28 smithi078 ceph-mon[27629]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:28 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:28.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:28 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:29.309 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:29 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:29.309 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:29 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:29.309 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:11:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:11:29] "GET /metrics HTTP/1.1" 200 38064 "" "Prometheus/2.43.0" 2024-05-09T04:11:29.310 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:29 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:29.310 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:29 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:29.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:29 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:29.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:29 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:30.248 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:30 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:11:30.082Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000032327s 2024-05-09T04:11:30.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:30 smithi132 ceph-mon[23807]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:30.506 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:30 smithi078 ceph-mon[27629]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:30.506 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:30 smithi078 ceph-mon[22905]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:32.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:32 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:32 smithi132 ceph-mon[23807]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:32.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:32 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:32.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:32 smithi078 ceph-mon[27629]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:32.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:32 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:32.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:32 smithi078 ceph-mon[22905]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:33.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:33 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:33.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:33 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:33.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:33 smithi132 ceph-mon[23807]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:33 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:33 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:33 smithi078 ceph-mon[22905]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:33 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:33 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:33 smithi078 ceph-mon[27629]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:34.841 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:11:35.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:34 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:35.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:34 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:35.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:34 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:35.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:34 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:34 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:34 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:34 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:34 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:35.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:34 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:35.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:34 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:35.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:34 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:35.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:34 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:35.520 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:11:35.520 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (7s) 2s ago 22m 16.4M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:11:35.520 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (14m) 1s ago 21m 53.8M - 8.3.5 dad864ee21e9 33b2a450dbe1 2024-05-09T04:11:35.520 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (85s) 2s ago 21m 52.7M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:11:35.520 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (75s) 1s ago 26m 465M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:11:35.520 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (11m) 2s ago 28m 518M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:11:35.520 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (28m) 2s ago 28m 78.2M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T04:11:35.520 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (27m) 1s ago 27m 61.1M 2048M 17.2.0 e1d6a67b021e a71bae52559d 2024-05-09T04:11:35.520 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (27m) 2s ago 27m 60.6M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T04:11:35.520 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (60s) 2s ago 22m 8404k - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:11:35.521 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (52s) 1s ago 22m 14.5M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:11:35.521 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (26m) 2s ago 26m 54.7M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:11:35.521 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (26m) 2s ago 26m 58.7M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:11:35.521 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (25m) 2s ago 25m 51.0M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:11:35.521 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (25m) 2s ago 25m 54.1M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:11:35.521 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (24m) 1s ago 24m 57.8M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:11:35.521 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (24m) 1s ago 24m 52.6M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:11:35.521 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (24m) 1s ago 24m 53.4M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:11:35.521 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (23m) 1s ago 23m 57.5M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:11:35.521 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (29s) 1s ago 23m 53.6M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:11:35.521 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (21m) 2s ago 21m 78.1M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:11:35.521 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (21m) 1s ago 21m 79.4M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:11:35.905 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:35 smithi132 ceph-mon[23807]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:35.905 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:35 smithi132 ceph-mon[23807]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:35.905 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:35 smithi132 ceph-mon[23807]: from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:35.909 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:35 smithi078 ceph-mon[27629]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:35.909 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:35 smithi078 ceph-mon[27629]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:35.909 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:35 smithi078 ceph-mon[27629]: from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:35.909 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:35 smithi078 ceph-mon[22905]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:35.909 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:35 smithi078 ceph-mon[22905]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:35.909 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:35 smithi078 ceph-mon[22905]: from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:35.937 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:11:35.938 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:11:35.938 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-09T04:11:35.938 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:11:35.938 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:11:35.938 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:11:35.939 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:11:35.939 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:11:35.939 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T04:11:35.939 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:11:35.939 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": {}, 2024-05-09T04:11:35.939 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:11:35.940 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:11:35.940 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:11:35.940 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:11:35.940 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-09T04:11:35.940 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:11:35.940 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:11:35.940 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:11:36.287 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:11:36.287 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:11:36.287 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:11:36.287 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-09T04:11:36.287 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-05-09T04:11:36.287 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-05-09T04:11:36.287 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-05-09T04:11:36.287 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/2 daemons upgraded", 2024-05-09T04:11:36.287 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading alertmanager daemons", 2024-05-09T04:11:36.287 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:11:36.287 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:11:36.926 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:36 smithi132 ceph-mon[23807]: from='client.25555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:36.950 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:36 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2262898333' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:36.950 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:36 smithi132 ceph-mon[23807]: from='client.25567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:36 smithi078 ceph-mon[22905]: from='client.25555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:36 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2262898333' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:36 smithi078 ceph-mon[22905]: from='client.25567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:37.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:36 smithi078 ceph-mon[27629]: from='client.25555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:37.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:36 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2262898333' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:37.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:36 smithi078 ceph-mon[27629]: from='client.25567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.283 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.284 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.284 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.284 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.284 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.284 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.284 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.511 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:11:38.085Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002695849s 2024-05-09T04:11:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:11:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:11:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T04:11:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:38.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:38.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:38.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:11:38.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:11:38.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:38.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T04:11:38.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:38.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:38.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:11:39.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:39 smithi132 ceph-mon[23807]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T04:11:39.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:39 smithi078 ceph-mon[22905]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T04:11:39.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:39 smithi078 ceph-mon[27629]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-09T04:11:39.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:11:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:11:39] "GET /metrics HTTP/1.1" 200 38064 "" "Prometheus/2.43.0" 2024-05-09T04:11:40.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:40 smithi078 ceph-mon[22905]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:40.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:40 smithi078 ceph-mon[22905]: Upgrade: Updating grafana.a 2024-05-09T04:11:40.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:40 smithi078 ceph-mon[22905]: Deploying daemon grafana.a on smithi132 2024-05-09T04:11:40.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:40 smithi078 ceph-mon[27629]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:40.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:40 smithi078 ceph-mon[27629]: Upgrade: Updating grafana.a 2024-05-09T04:11:40.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:40 smithi078 ceph-mon[27629]: Deploying daemon grafana.a on smithi132 2024-05-09T04:11:40.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:40 smithi132 ceph-mon[23807]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:40.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:40 smithi132 ceph-mon[23807]: Upgrade: Updating grafana.a 2024-05-09T04:11:40.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:40 smithi132 ceph-mon[23807]: Deploying daemon grafana.a on smithi132 2024-05-09T04:11:42.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:42 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:42.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:42 smithi078 ceph-mon[27629]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:42 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:42 smithi078 ceph-mon[22905]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:42.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:42 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:42.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:42 smithi132 ceph-mon[23807]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:44.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:44 smithi078 ceph-mon[22905]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:44.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:44 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:44.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:44 smithi078 ceph-mon[27629]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:44.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:44 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:44.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:44 smithi132 ceph-mon[23807]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:44.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:44 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:46.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:46 smithi078 ceph-mon[22905]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:46 smithi078 ceph-mon[27629]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:46.585 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:46 smithi132 ceph-mon[23807]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:48.463 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:48 smithi132 ceph-mon[23807]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:48.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:48 smithi078 ceph-mon[22905]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:48.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:48 smithi078 ceph-mon[27629]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:48.976 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:48 smithi132 systemd[1]: Stopping Ceph grafana.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:11:49.252 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:48 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[47740]: t=2024-05-09T04:11:48+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-05-09T04:11:49.252 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:48 smithi132 podman[64561]: 2024-05-09 04:11:48.986387622 +0000 UTC m=+0.124259262 container died 33b2a450dbe16d5881623fba7157c2dc56aa81948aa49aaabdbffd738a1fa12e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, vendor=Red Hat, Inc., description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, distribution-scope=public, architecture=x86_64, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, maintainer=Paul Cuzner , vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, version=8.5, build-date=2022-03-28T10:36:18.413762, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-05-09T04:11:49.503 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:49 smithi132 podman[64561]: 2024-05-09 04:11:49.350847124 +0000 UTC m=+0.488718760 container cleanup 33b2a450dbe16d5881623fba7157c2dc56aa81948aa49aaabdbffd738a1fa12e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, description=Ceph Grafana Container, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.24.2, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, io.openshift.tags=base rhel8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vcs-type=git, vendor=Red Hat, Inc., maintainer=Paul Cuzner , com.redhat.component=ubi8-container) 2024-05-09T04:11:49.503 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:49 smithi132 bash[64561]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a 2024-05-09T04:11:49.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:11:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:11:49] "GET /metrics HTTP/1.1" 200 38067 "" "Prometheus/2.43.0" 2024-05-09T04:11:49.923 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:49 smithi132 podman[64575]: 2024-05-09 04:11:49.593847529 +0000 UTC m=+0.608626029 container remove 33b2a450dbe16d5881623fba7157c2dc56aa81948aa49aaabdbffd738a1fa12e (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, release=236.1648460182, distribution-scope=public, version=8.5, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.buildah.version=1.24.2, io.openshift.tags=base rhel8, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , vcs-type=git, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-09T04:11:49.923 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:49 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@grafana.a.service: Deactivated successfully. 2024-05-09T04:11:49.923 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:49 smithi132 systemd[1]: Stopped Ceph grafana.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:11:49.923 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:49 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@grafana.a.service: Consumed 5.595s CPU time. 2024-05-09T04:11:50.245 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:49 smithi132 systemd[1]: Starting Ceph grafana.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:11:50.246 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 podman[64707]: 2024-05-09 04:11:50.136844904 +0000 UTC m=+0.111802082 container create 663d68ed82f1d37c3e0922fe9df5a0554d9fbac8d0cc0dae68686069b28e7f25 (image=quay.io/ceph/grafana:9.4.12, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-09T04:11:50.246 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 podman[64707]: 2024-05-09 04:11:50.043687037 +0000 UTC m=+0.018644214 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-05-09T04:11:50.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:50 smithi132 ceph-mon[23807]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:50.504 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 podman[64707]: 2024-05-09 04:11:50.380872202 +0000 UTC m=+0.355829383 container init 663d68ed82f1d37c3e0922fe9df5a0554d9fbac8d0cc0dae68686069b28e7f25 (image=quay.io/ceph/grafana:9.4.12, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-09T04:11:50.504 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 podman[64707]: 2024-05-09 04:11:50.384024895 +0000 UTC m=+0.358982081 container start 663d68ed82f1d37c3e0922fe9df5a0554d9fbac8d0cc0dae68686069b28e7f25 (image=quay.io/ceph/grafana:9.4.12, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-09T04:11:50.504 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 bash[64707]: 663d68ed82f1d37c3e0922fe9df5a0554d9fbac8d0cc0dae68686069b28e7f25 2024-05-09T04:11:50.504 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 systemd[1]: Started Ceph grafana.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:11:50.526 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:50 smithi078 ceph-mon[27629]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:50.526 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:50 smithi078 ceph-mon[22905]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:50.804 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.533824506Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-05-09T04:11:50.804 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534004976Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-05-09T04:11:50.804 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534016074Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-05-09T04:11:50.804 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534019604Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-05-09T04:11:50.804 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534022889Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-05-09T04:11:50.804 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534025669Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-05-09T04:11:50.805 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534028281Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-05-09T04:11:50.805 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534030722Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-05-09T04:11:50.805 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534033944Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-05-09T04:11:50.805 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534036763Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-05-09T04:11:50.805 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534039831Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-05-09T04:11:50.805 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534042243Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-05-09T04:11:50.805 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534044752Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-05-09T04:11:50.805 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534047374Z level=info msg="Path Home" path=/usr/share/grafana 2024-05-09T04:11:50.805 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534050014Z level=info msg="Path Data" path=/var/lib/grafana 2024-05-09T04:11:50.805 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534052498Z level=info msg="Path Logs" path=/var/log/grafana 2024-05-09T04:11:50.805 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534054814Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-05-09T04:11:50.805 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534057251Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-05-09T04:11:50.806 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=settings t=2024-05-09T04:11:50.534059993Z level=info msg="App mode production" 2024-05-09T04:11:50.806 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=sqlstore t=2024-05-09T04:11:50.534099246Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-05-09T04:11:50.806 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=sqlstore t=2024-05-09T04:11:50.534110702Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-05-09T04:11:50.806 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.545765766Z level=info msg="Starting DB migrations" 2024-05-09T04:11:50.806 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.549000699Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-05-09T04:11:50.806 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.560240044Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-05-09T04:11:50.806 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.561376652Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.561731489Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.564573556Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.567351772Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.570056499Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.572896496Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.575599534Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.578168737Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.580875481Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.583679198Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.586045072Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.588716084Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.591486743Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.594075139Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.596772063Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.599489867Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.602032075Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-05-09T04:11:50.807 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.6048938Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.607482346Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.610162954Z level=info msg="Executing migration" id="create provenance_type table" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.612846168Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.61530619Z level=info msg="Executing migration" id="create alert_image table" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.618086609Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.618846278Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.619702311Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.620547092Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.621371892Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.621457569Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.622232549Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.623072926Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.623939194Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.624698698Z level=info msg="Executing migration" id="create secrets table" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.625590715Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-05-09T04:11:50.808 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.631814054Z level=info msg="Executing migration" id="add name column into data_keys" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.632980024Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.633829563Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.640093494Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.646406091Z level=info msg="Executing migration" id="add column hidden to role table" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.64772794Z level=info msg="Executing migration" id="create query_history table v1" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.64837039Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.649232784Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.650065452Z level=info msg="Executing migration" id="teams permissions migration" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.6509464Z level=info msg="Executing migration" id="dashboard permissions" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.65225947Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.653177193Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.654019205Z level=info msg="Executing migration" id="alerting notification permissions" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.654795714Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.655639332Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.656487723Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-05-09T04:11:50.809 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.657656785Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-05-09T04:11:50.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.658495904Z level=info msg="Executing migration" id="create correlation table v1" 2024-05-09T04:11:50.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.659339176Z level=info msg="Executing migration" id="add index correlations.uid" 2024-05-09T04:11:50.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.660146253Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-05-09T04:11:50.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.66103299Z level=info msg="Executing migration" id="add correlation config column" 2024-05-09T04:11:50.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.662179946Z level=info msg="Executing migration" id="create entity_events table" 2024-05-09T04:11:50.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.663021513Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-05-09T04:11:50.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.663905829Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-05-09T04:11:50.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.663996272Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-05-09T04:11:50.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.664777609Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-05-09T04:11:50.814 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.664867662Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.665732185Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.668227085Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.670903841Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.673673823Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.676491736Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.679092708Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.681872585Z level=info msg="Executing migration" id="Drop public config table" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.68472761Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.687541264Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.690244151Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.692989395Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.695659178Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.69957893Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.702384061Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-05-09T04:11:50.815 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.705188046Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.708069285Z level=info msg="Executing migration" id="add share column" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.710933894Z level=info msg="Executing migration" id="create default alerting folders" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.713836525Z level=info msg="Executing migration" id="create file table" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.716285556Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.719034717Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.721759947Z level=info msg="Executing migration" id="create file_meta table" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.724302882Z level=info msg="Executing migration" id="file table idx: path key" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.727076874Z level=info msg="Executing migration" id="set path collation in file table" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.729656697Z level=info msg="Executing migration" id="managed permissions migration" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.732250822Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.734843319Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.737551178Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.740234983Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-05-09T04:11:50.816 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.740551574Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.740989908Z level=info msg="Executing migration" id="update group index for alert rules" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.741417327Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.741954631Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.742317501Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.743588747Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.744787101Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.747473065Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.760815085Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.763479585Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.763950319Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.768330882Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.768832093Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.769155081Z level=info msg="Executing migration" id="create folder table" 2024-05-09T04:11:50.817 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.769567715Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-05-09T04:11:50.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.770001223Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-05-09T04:11:50.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.770444433Z level=info msg="Executing migration" id="Update folder title length" 2024-05-09T04:11:50.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.770727444Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-05-09T04:11:50.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=migrator t=2024-05-09T04:11:50.771173287Z level=info msg="migrations completed" performed=105 skipped=376 duration=222.206247ms 2024-05-09T04:11:50.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=sqlstore t=2024-05-09T04:11:50.77149509Z level=info msg="Created default organization" 2024-05-09T04:11:50.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=plugin.finder t=2024-05-09T04:11:50.804186483Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-05-09T04:11:50.818 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=secrets t=2024-05-09T04:11:50.804305725Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-05-09T04:11:51.253 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=query_data t=2024-05-09T04:11:50.809312896Z level=info msg="Query Service initialization" 2024-05-09T04:11:51.253 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=live.push_http t=2024-05-09T04:11:50.811836519Z level=info msg="Live Push Gateway initialization" 2024-05-09T04:11:51.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:51 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:51 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:51 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:51 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:51.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:51 smithi132 ceph-mon[23807]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:51.755 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=infra.usagestats.collector t=2024-05-09T04:11:51.517645227Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-05-09T04:11:51.755 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=provisioning.datasources t=2024-05-09T04:11:51.519763213Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-05-09T04:11:51.755 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=provisioning.datasources t=2024-05-09T04:11:51.519774643Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-05-09T04:11:51.755 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=provisioning.datasources t=2024-05-09T04:11:51.519879137Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-05-09T04:11:51.755 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=provisioning.alerting t=2024-05-09T04:11:51.542843021Z level=info msg="starting to provision alerting" 2024-05-09T04:11:51.755 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=provisioning.alerting t=2024-05-09T04:11:51.54285851Z level=info msg="finished to provision alerting" 2024-05-09T04:11:51.755 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=ngalert.state.manager t=2024-05-09T04:11:51.543014548Z level=info msg="Warming state cache for startup" 2024-05-09T04:11:51.755 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=grafanaStorageLogger t=2024-05-09T04:11:51.543128325Z level=info msg="storage starting" 2024-05-09T04:11:51.756 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=http.server t=2024-05-09T04:11:51.545532331Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-05-09T04:11:51.756 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=ngalert.state.manager t=2024-05-09T04:11:51.558474734Z level=info msg="State cache has been initialized" states=0 duration=15.457108ms 2024-05-09T04:11:51.756 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=ticker t=2024-05-09T04:11:51.558562022Z level=info msg=starting first_tick=2024-05-09T04:12:00Z 2024-05-09T04:11:51.756 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=ngalert.multiorg.alertmanager t=2024-05-09T04:11:51.558555909Z level=info msg="starting MultiOrg Alertmanager" 2024-05-09T04:11:51.756 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=sqlstore.transactions t=2024-05-09T04:11:51.56673851Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-05-09T04:11:51.756 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=sqlstore.transactions t=2024-05-09T04:11:51.577216707Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-05-09T04:11:51.756 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:11:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=sqlstore.transactions t=2024-05-09T04:11:51.588381969Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-05-09T04:11:51.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:51 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:51.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:51 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:51.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:51 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:51.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:51 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:51.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:51 smithi078 ceph-mon[22905]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:51.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:51 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:51.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:51 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:11:51.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:51 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:51.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:51 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:11:51.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:51 smithi078 ceph-mon[27629]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:54.482 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:54 smithi132 ceph-mon[23807]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:54.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:54 smithi078 ceph-mon[22905]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:54.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:54 smithi078 ceph-mon[27629]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:56.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:55 smithi132 ceph-mon[23807]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:56.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:55 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:56.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:55 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:56.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:55 smithi078 ceph-mon[22905]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:56.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:55 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:56.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:55 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:56.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:55 smithi078 ceph-mon[27629]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:11:56.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:55 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:56.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:55 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:57.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:56 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:57.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:56 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:57.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:56 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:57.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:56 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:57.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:56 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:57.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:56 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:11:58.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:57 smithi132 ceph-mon[23807]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:58.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:57 smithi078 ceph-mon[22905]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:58.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:57 smithi078 ceph-mon[27629]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:11:59.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:11:59 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:11:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:59.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:11:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:11:59.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:11:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:11:59] "GET /metrics HTTP/1.1" 200 38066 "" "Prometheus/2.43.0" 2024-05-09T04:12:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T04:12:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:12:00.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:12:00.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:12:00.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:12:00.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:12:00.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:12:00.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:12:00.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:12:00.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:12:00.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:12:00.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:00.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:00.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:00.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:00.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:00.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:00.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:00 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T04:12:00.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:12:00.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:12:00.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:12:00.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:12:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T04:12:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:12:00.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:12:00.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:12:00.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:12:00.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:12:00.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:12:00.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:12:00.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:12:00.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:12:00.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:12:00.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:12:00.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:12:00.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:12:00.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:12:00.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:12:00.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:12:00.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:12:00.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:12:00.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:12:00.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:12:00.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:12:00.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:12:00.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:12:00.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:12:00.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:00.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:12:00.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:12:00.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:00.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:00.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:00.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:00.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:00.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:00.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.522 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:12:00.522 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:12:00.522 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:00.522 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:00.522 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:00.522 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:00.522 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:00.522 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:00.522 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:00.522 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:00 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:01.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:01 smithi132 ceph-mon[23807]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T04:12:01.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:01 smithi132 ceph-mon[23807]: Upgrade: Finalizing container_image settings 2024-05-09T04:12:01.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:01 smithi132 ceph-mon[23807]: Upgrade: Complete! 2024-05-09T04:12:01.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:01 smithi078 ceph-mon[22905]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T04:12:01.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:01 smithi078 ceph-mon[22905]: Upgrade: Finalizing container_image settings 2024-05-09T04:12:01.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:01 smithi078 ceph-mon[22905]: Upgrade: Complete! 2024-05-09T04:12:01.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:01 smithi078 ceph-mon[27629]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-09T04:12:01.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:01 smithi078 ceph-mon[27629]: Upgrade: Finalizing container_image settings 2024-05-09T04:12:01.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:01 smithi078 ceph-mon[27629]: Upgrade: Complete! 2024-05-09T04:12:02.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:02 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:02.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:02 smithi132 ceph-mon[23807]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:02.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:02 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:02.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:02 smithi078 ceph-mon[27629]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:02.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:02 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:02.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:02 smithi078 ceph-mon[22905]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:04.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:04 smithi132 ceph-mon[23807]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:04.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:04 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:04.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:04 smithi078 ceph-mon[27629]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:04.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:04 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:04.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:04 smithi078 ceph-mon[22905]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:04.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:04 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:06.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:06 smithi078 ceph-mon[22905]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:06.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:06 smithi078 ceph-mon[27629]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:06.557 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:06 smithi132 ceph-mon[23807]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:07.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-05-09T04:12:08.448 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:08 smithi078 ceph-mon[27629]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:08.448 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:08 smithi078 ceph-mon[27629]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:08.449 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:08 smithi078 ceph-mon[22905]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:08.449 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:08 smithi078 ceph-mon[22905]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:08 smithi132 ceph-mon[23807]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:08.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:08 smithi132 ceph-mon[23807]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:09.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:12:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:12:09] "GET /metrics HTTP/1.1" 200 38066 "" "Prometheus/2.43.0" 2024-05-09T04:12:09.541 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:12:10.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-05-09T04:12:10.431 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:10 smithi078 ceph-mon[27629]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:10.431 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:10 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/447057053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:10.431 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:10 smithi078 ceph-mon[22905]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:10.431 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:10 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/447057053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:10.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:10 smithi132 ceph-mon[23807]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:10 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/447057053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:12.249 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)" 2024-05-09T04:12:12.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:12 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:12.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:12 smithi078 ceph-mon[22905]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:12.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:12 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2552589915' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:12.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:12 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:12.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:12 smithi078 ceph-mon[27629]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:12.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:12 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2552589915' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:12.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:12 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:12.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:12 smithi132 ceph-mon[23807]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:12.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:12 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2552589915' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:12.839 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-05-09T04:12:14.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:14 smithi078 ceph-mon[27629]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:14.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:14 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:12:14.361 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:14 smithi078 ceph-mon[22905]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:14.361 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:14 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:12:14.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:14 smithi132 ceph-mon[23807]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:14.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:14 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:12:14.905 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:12:15.491 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-05-09T04:12:15.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:15 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/280238764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:15.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:15 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/280238764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:15.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:15 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/280238764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:16.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:16 smithi078 ceph-mon[22905]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:16.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:16 smithi078 ceph-mon[27629]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:16.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:16 smithi132 ceph-mon[23807]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:18.536 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:18 smithi078 ceph-mon[22905]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:18.537 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:18 smithi078 ceph-mon[27629]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:18.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:18 smithi132 ceph-mon[23807]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:19.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:12:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:12:19] "GET /metrics HTTP/1.1" 200 38075 "" "Prometheus/2.43.0" 2024-05-09T04:12:19.483 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:12:19.571 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:19 smithi078 ceph-mon[22905]: from='client.25597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:19.571 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:19 smithi078 ceph-mon[27629]: from='client.25597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:19.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:19 smithi132 ceph-mon[23807]: from='client.25597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:20.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade status' 2024-05-09T04:12:20.477 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:20 smithi078 ceph-mon[22905]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:20.478 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:20 smithi078 ceph-mon[27629]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:20.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:20 smithi132 ceph-mon[23807]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:22.053 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:12:22.053 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": null, 2024-05-09T04:12:22.053 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": false, 2024-05-09T04:12:22.053 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "", 2024-05-09T04:12:22.053 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:12:22.053 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": null, 2024-05-09T04:12:22.053 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-05-09T04:12:22.053 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:12:22.053 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:12:22.379 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:22 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:22.379 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:22 smithi078 ceph-mon[22905]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:22.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:22 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:22.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:22 smithi078 ceph-mon[27629]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:22.704 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph health detail' 2024-05-09T04:12:22.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:22 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:22.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:22 smithi132 ceph-mon[23807]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:23.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:23 smithi132 ceph-mon[23807]: from='client.25603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:23.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:23 smithi078 ceph-mon[22905]: from='client.25603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:23.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:23 smithi078 ceph-mon[27629]: from='client.25603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:24.577 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:24 smithi078 ceph-mon[22905]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:24.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:24 smithi078 ceph-mon[27629]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:24.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:24 smithi132 ceph-mon[23807]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:24.895 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-05-09T04:12:25.489 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:25 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2895180606' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:12:25.489 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:25 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2895180606' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:12:25.513 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-05-09T04:12:25.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:25 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2895180606' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:12:26.479 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:26 smithi078 ceph-mon[27629]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:26.479 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:26 smithi078 ceph-mon[22905]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:26.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:26 smithi132 ceph-mon[23807]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:27.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:27 smithi132 ceph-mon[23807]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:27.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:27 smithi078 ceph-mon[22905]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:27.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:27 smithi078 ceph-mon[27629]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:29.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:29 smithi132 ceph-mon[23807]: from='client.25615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:29.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:29 smithi132 ceph-mon[23807]: from='client.15720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "mon", "hosts": "smithi132", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:29.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:29 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:12:29.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:12:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:12:29] "GET /metrics HTTP/1.1" 200 38073 "" "Prometheus/2.43.0" 2024-05-09T04:12:29.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:29 smithi078 ceph-mon[22905]: from='client.25615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:29.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:29 smithi078 ceph-mon[22905]: from='client.15720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "mon", "hosts": "smithi132", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:29.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:29 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:12:29.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:29 smithi078 ceph-mon[27629]: from='client.25615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:29.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:29 smithi078 ceph-mon[27629]: from='client.15720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "mon", "hosts": "smithi132", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:29.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:29 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:12:30.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:30 smithi132 ceph-mon[23807]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:30.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:30 smithi078 ceph-mon[22905]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:30.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:30 smithi078 ceph-mon[27629]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:32.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:32 smithi132 ceph-mon[23807]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:32.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:32 smithi132 ceph-mon[23807]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:32.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:32 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:32.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:32 smithi078 ceph-mon[22905]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:32.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:32 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:32.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:32 smithi078 ceph-mon[27629]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:32.735 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:12:33.392 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-09T04:12:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:33 smithi132 ceph-mon[23807]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:33 smithi132 ceph-mon[23807]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:12:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:33 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:33 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:33 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:34.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:33 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:34.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:33 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:34.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:33 smithi132 ceph-mon[23807]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:12:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[22905]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[22905]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:12:34.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:34.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:34.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:34.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:34.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:34.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[22905]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:12:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[27629]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[27629]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:12:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:34.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:34.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:34.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:33 smithi078 ceph-mon[27629]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:12:35.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[27629]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:35.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:35.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:35.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:35.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:35.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-09T04:12:35.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-05-09T04:12:35.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[22905]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:35.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:35.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:35.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:35.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:35.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-09T04:12:35.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:35 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-05-09T04:12:36.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:35 smithi132 ceph-mon[23807]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:36.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:35 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:36.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:35 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:35 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:35 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:35 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-09T04:12:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:35 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-05-09T04:12:36.592 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:12:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:36 smithi132 ceph-mon[23807]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:12:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:36 smithi132 ceph-mon[23807]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:12:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:36 smithi132 ceph-mon[23807]: Upgrade: Setting container_image for all mgr 2024-05-09T04:12:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:36 smithi132 ceph-mon[23807]: Upgrade: It appears safe to stop mon.b 2024-05-09T04:12:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:36 smithi078 ceph-mon[22905]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:12:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:36 smithi078 ceph-mon[22905]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:12:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:36 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all mgr 2024-05-09T04:12:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:36 smithi078 ceph-mon[22905]: Upgrade: It appears safe to stop mon.b 2024-05-09T04:12:37.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:36 smithi078 ceph-mon[27629]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:12:37.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:36 smithi078 ceph-mon[27629]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:12:37.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:36 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all mgr 2024-05-09T04:12:37.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:36 smithi078 ceph-mon[27629]: Upgrade: It appears safe to stop mon.b 2024-05-09T04:12:37.261 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:12:37.261 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (69s) 64s ago 23m 16.4M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:12:37.261 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (47s) 42s ago 22m 48.8M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:12:37.261 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (2m) 64s ago 22m 52.7M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:12:37.261 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (2m) 42s ago 27m 465M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:12:37.261 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (12m) 64s ago 29m 518M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (29m) 64s ago 29m 78.2M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (28m) 42s ago 28m 61.2M 2048M 17.2.0 e1d6a67b021e a71bae52559d 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (28m) 64s ago 28m 60.6M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (2m) 64s ago 24m 8404k - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (114s) 42s ago 23m 16.7M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (27m) 64s ago 27m 54.7M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (27m) 64s ago 27m 58.7M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (26m) 64s ago 26m 51.0M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (26m) 64s ago 26m 54.1M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (25m) 42s ago 25m 57.8M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (25m) 42s ago 25m 52.6M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (25m) 42s ago 25m 53.4M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (24m) 42s ago 24m 57.5M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:12:37.262 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (90s) 42s ago 24m 54.1M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:12:37.263 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (22m) 64s ago 22m 78.1M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:12:37.263 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (22m) 42s ago 22m 79.5M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:12:37.673 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:12:37.673 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:12:37.673 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-09T04:12:37.673 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:12:37.673 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:12:37.673 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:12:37.673 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:12:37.673 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:12:37.673 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T04:12:37.673 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:12:37.674 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": {}, 2024-05-09T04:12:37.674 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:12:37.674 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:12:37.674 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:12:37.674 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:12:37.674 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-09T04:12:37.674 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:12:37.674 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:12:37.674 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:12:38.014 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:12:38.014 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:12:38.015 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:12:38.015 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi132", 2024-05-09T04:12:38.015 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:12:38.015 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "0/1 daemons upgraded", 2024-05-09T04:12:38.015 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mon daemons", 2024-05-09T04:12:38.015 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:12:38.015 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:12:38.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:38 smithi132 ceph-mon[23807]: from='client.25627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:38 smithi132 ceph-mon[23807]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:12:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:38 smithi132 ceph-mon[23807]: from='client.25633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:38 smithi132 ceph-mon[23807]: from='client.25639 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:38 smithi132 ceph-mon[23807]: Upgrade: Updating mon.b 2024-05-09T04:12:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:12:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:12:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:38 smithi132 ceph-mon[23807]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:38 smithi132 ceph-mon[23807]: Deploying daemon mon.b on smithi132 2024-05-09T04:12:38.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:38 smithi132 ceph-mon[23807]: from='client.? 172.21.15.78:0/2861622819' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:38.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[27629]: from='client.25627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[27629]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:12:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[27629]: from='client.25633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[27629]: from='client.25639 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[27629]: Upgrade: Updating mon.b 2024-05-09T04:12:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:12:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:12:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[27629]: Deploying daemon mon.b on smithi132 2024-05-09T04:12:38.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2861622819' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[22905]: from='client.25627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[22905]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:12:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[22905]: from='client.25633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[22905]: from='client.25639 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[22905]: Upgrade: Updating mon.b 2024-05-09T04:12:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:12:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:12:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[22905]: Deploying daemon mon.b on smithi132 2024-05-09T04:12:38.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:38 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2861622819' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:39.441 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:39 smithi132 systemd[1]: Stopping Ceph mon.b for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:12:39.441 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:39 smithi132 ceph-mon[23807]: from='client.15750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:39.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:39 smithi078 ceph-mon[27629]: from='client.15750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:39.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:39 smithi078 ceph-mon[22905]: from='client.15750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:12:39.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:12:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:12:39] "GET /metrics HTTP/1.1" 200 38073 "" "Prometheus/2.43.0" 2024-05-09T04:12:39.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b[23789]: 2024-05-09T04:12:39.439+0000 7fe378469700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:12:39.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b[23789]: 2024-05-09T04:12:39.439+0000 7fe378469700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-05-09T04:12:39.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:39 smithi132 podman[66182]: 2024-05-09 04:12:39.531891274 +0000 UTC m=+0.221649425 container died a71bae52559df4ee8bff01c3c97e57148a5927605408fbf72ccd94fbceb27387 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b, architecture=x86_64, com.redhat.component=centos-stream-container, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, version=8, distribution-scope=public, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, release=754, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-09T04:12:40.174 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:39 smithi132 podman[66182]: 2024-05-09 04:12:39.923408509 +0000 UTC m=+0.613166683 container cleanup a71bae52559df4ee8bff01c3c97e57148a5927605408fbf72ccd94fbceb27387 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, release=754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-09T04:12:40.175 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:39 smithi132 bash[66182]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b 2024-05-09T04:12:40.501 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:40 smithi132 podman[66196]: 2024-05-09 04:12:40.174989251 +0000 UTC m=+0.634847911 container remove a71bae52559df4ee8bff01c3c97e57148a5927605408fbf72ccd94fbceb27387 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, name=centos-stream, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, vcs-type=git, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, version=8, GIT_CLEAN=True, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True) 2024-05-09T04:12:40.501 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:40 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.b.service: Deactivated successfully. 2024-05-09T04:12:40.501 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:40 smithi132 systemd[1]: Stopped Ceph mon.b for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:12:40.501 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:40 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.b.service: Consumed 17.315s CPU time. 2024-05-09T04:12:40.501 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:40 smithi132 systemd[1]: Starting Ceph mon.b for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:12:40.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:40 smithi132 podman[66328]: 2024-05-09 04:12:40.62250658 +0000 UTC m=+0.018435937 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:12:40.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:40 smithi132 podman[66328]: 2024-05-09 04:12:40.742877128 +0000 UTC m=+0.138806519 container create d9d8fb1d89f08541cf7511ef2eabf18ac7825d1d3191d965c3abce7b0e534115 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b, GIT_CLEAN=True, org.label-schema.build-date=20240507, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:12:41.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 podman[66328]: 2024-05-09 04:12:41.003994291 +0000 UTC m=+0.399923637 container init d9d8fb1d89f08541cf7511ef2eabf18ac7825d1d3191d965c3abce7b0e534115 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:12:41.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 podman[66328]: 2024-05-09 04:12:41.007320757 +0000 UTC m=+0.403250106 container start d9d8fb1d89f08541cf7511ef2eabf18ac7825d1d3191d965c3abce7b0e534115 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-09T04:12:41.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: set uid:gid to 167:167 (ceph:ceph) 2024-05-09T04:12:41.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev), process ceph-mon, pid 2 2024-05-09T04:12:41.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: pidfile_write: ignore empty --pid-file 2024-05-09T04:12:41.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: load: jerasure load: lrc 2024-05-09T04:12:41.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: RocksDB version: 7.9.2 2024-05-09T04:12:41.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Git sha 0 2024-05-09T04:12:41.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Compile date 2024-05-08 23:53:35 2024-05-09T04:12:41.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: DB SUMMARY 2024-05-09T04:12:41.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: DB Session ID: 5R6QFVO3BYF0TGVP2FFG 2024-05-09T04:12:41.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: CURRENT file: CURRENT 2024-05-09T04:12:41.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: IDENTITY file: IDENTITY 2024-05-09T04:12:41.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2813 Bytes 2024-05-09T04:12:41.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-05-09T04:12:41.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 2550802 ; 2024-05-09T04:12:41.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.error_if_exists: 0 2024-05-09T04:12:41.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.create_if_missing: 0 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.paranoid_checks: 1 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.env: 0x564b1bf80cc0 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.fs: PosixFileSystem 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.info_log: 0x564b1ceb9dc0 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_file_opening_threads: 16 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.statistics: (nil) 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.use_fsync: 0 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_log_file_size: 0 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.keep_log_file_num: 1000 2024-05-09T04:12:41.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.recycle_log_file_num: 0 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.allow_fallocate: 1 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.allow_mmap_reads: 0 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.allow_mmap_writes: 0 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.use_direct_reads: 0 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.create_missing_column_families: 0 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.db_log_dir: 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.wal_dir: 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.advise_random_on_open: 1 2024-05-09T04:12:41.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.db_write_buffer_size: 0 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.write_buffer_manager: 0x564b1ce8b220 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.rate_limiter: (nil) 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.wal_recovery_mode: 2 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.enable_thread_tracking: 0 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.enable_pipelined_write: 0 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.unordered_write: 0 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.row_cache: None 2024-05-09T04:12:41.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.wal_filter: None 2024-05-09T04:12:41.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.allow_ingest_behind: 0 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.two_write_queues: 0 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.manual_wal_flush: 0 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.wal_compression: 0 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.atomic_flush: 0 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.log_readahead_size: 0 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.best_efforts_recovery: 0 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.allow_data_in_errors: 0 2024-05-09T04:12:41.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.db_host_id: __hostname__ 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_background_jobs: 2 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_background_compactions: -1 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_subcompactions: 1 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_total_wal_size: 0 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_open_files: -1 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bytes_per_sync: 0 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_readahead_size: 0 2024-05-09T04:12:41.261 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_background_flushes: -1 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Compression algorithms supported: 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: kZSTD supported: 0 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: kXpressCompression supported: 0 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: kLZ4HCCompression supported: 1 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: kZlibCompression supported: 1 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: kSnappyCompression supported: 1 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: kLZ4Compression supported: 1 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: kBZip2Compression supported: 0 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-09T04:12:41.262 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.merge_operator: 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_filter: None 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_filter_factory: None 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.sst_partitioner_factory: None 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564b1ceb9f80) 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout: cache_index_and_filter_blocks: 1 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout: pin_top_level_index_and_filter: 1 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout: index_type: 0 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout: data_block_index_type: 0 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout: index_shortening: 1 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout: checksum: 4 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout: no_block_cache: 0 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_cache: 0x564b1cebd1f0 2024-05-09T04:12:41.263 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_cache_name: BinnedLRUCache 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_cache_options: 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: capacity : 536870912 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: num_shard_bits : 4 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: strict_capacity_limit : 0 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: high_pri_pool_ratio: 0.000 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_cache_compressed: (nil) 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: persistent_cache: (nil) 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_size: 4096 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_size_deviation: 10 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_restart_interval: 16 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: index_block_restart_interval: 1 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: metadata_block_size: 4096 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: partition_filters: 0 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: use_delta_encoding: 1 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: filter_policy: bloomfilter 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: whole_key_filtering: 1 2024-05-09T04:12:41.264 INFO:journalctl@ceph.mon.b.smithi132.stdout: verify_compression: 0 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout: read_amp_bytes_per_bit: 0 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout: format_version: 5 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout: enable_index_compression: 1 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout: block_align: 0 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout: max_auto_readahead_size: 262144 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout: prepopulate_block_cache: 0 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout: initial_auto_readahead_size: 8192 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout: num_file_reads_for_auto_readahead: 2 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.write_buffer_size: 33554432 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_write_buffer_number: 2 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compression: NoCompression 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bottommost_compression: Disabled 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.prefix_extractor: nullptr 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.num_levels: 7 2024-05-09T04:12:41.265 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compression_opts.level: 32767 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compression_opts.strategy: 0 2024-05-09T04:12:41.266 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compression_opts.enabled: false 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.target_file_size_base: 67108864 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-09T04:12:41.267 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.arena_block_size: 1048576 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.disable_auto_compactions: 0 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-09T04:12:41.268 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.inplace_update_support: 0 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.bloom_locality: 0 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.max_successive_merges: 0 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.paranoid_file_checks: 0 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.force_consistency_checks: 1 2024-05-09T04:12:41.269 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.report_bg_io_stats: 0 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.ttl: 2592000 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.enable_blob_files: false 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.min_blob_size: 0 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.blob_file_size: 268435456 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.blob_file_starting_level: 0 2024-05-09T04:12:41.270 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28798, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 75af080e-61ce-4a38-a278-0ede93e52399 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715227961039229, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715227961048102, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 1569276, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28803, "largest_seqno": 30335, "table_properties": {"data_size": 1563350, "index_size": 3413, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 13983, "raw_average_key_size": 23, "raw_value_size": 1551122, "raw_average_value_size": 2656, "num_data_blocks": 154, "num_entries": 584, "num_filter_entries": 584, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1715227961, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "75af080e-61ce-4a38-a278-0ede93e52399", "db_session_id": "5R6QFVO3BYF0TGVP2FFG", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715227961048176, "job": 1, "event": "recovery_finished"} 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564b1cfda000 2024-05-09T04:12:41.271 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: DB pointer 0x564b1cfc6000 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: ** DB Stats ** 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: ** Compaction Stats [default] ** 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: L0 1/0 1.50 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 487.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-09T04:12:41.272 INFO:journalctl@ceph.mon.b.smithi132.stdout: L6 1/0 8.89 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: Sum 2/0 10.38 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 487.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 487.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: ** Compaction Stats [default] ** 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 487.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-09T04:12:41.273 INFO:journalctl@ceph.mon.b.smithi132.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout: Cumulative compaction: 0.00 GB write, 110.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout: Interval compaction: 0.00 GB write, 110.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout: Block cache BinnedLRUCache@0x564b1cebd1f0#2 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: starting mon.b rank 2 at public addrs [v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0] at bind addrs [v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: mon.b@-1(???) e3 preinit fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: mon.b@-1(???).mds e1 new map 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: mon.b@-1(???).mds e1 print_map 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout: e1 2024-05-09T04:12:41.274 INFO:journalctl@ceph.mon.b.smithi132.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-09T04:12:41.275 INFO:journalctl@ceph.mon.b.smithi132.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-09T04:12:41.275 INFO:journalctl@ceph.mon.b.smithi132.stdout: legacy client fscid: -1 2024-05-09T04:12:41.275 INFO:journalctl@ceph.mon.b.smithi132.stdout: 2024-05-09T04:12:41.275 INFO:journalctl@ceph.mon.b.smithi132.stdout: No filesystems configured 2024-05-09T04:12:41.275 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: mon.b@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-09T04:12:41.275 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-05-09T04:12:41.275 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-05-09T04:12:41.275 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-05-09T04:12:41.275 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: mon.b@-1(???).paxosservice(auth 1..25) refresh upgraded, format 0 -> 3 2024-05-09T04:12:41.275 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 bash[66328]: d9d8fb1d89f08541cf7511ef2eabf18ac7825d1d3191d965c3abce7b0e534115 2024-05-09T04:12:41.275 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 systemd[1]: Started Ceph mon.b for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:12:41.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:41 smithi132 ceph-mon[66347]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:12:41.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:41 smithi078 ceph-mon[27629]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:12:41.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:41 smithi078 ceph-mon[22905]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:12:42.395 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:42.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:42.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: mon.b calling monitor election 2024-05-09T04:12:42.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: mon.c calling monitor election 2024-05-09T04:12:42.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: mon.a calling monitor election 2024-05-09T04:12:42.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-09T04:12:42.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} 2024-05-09T04:12:42.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: fsmap 2024-05-09T04:12:42.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: osdmap e112: 8 total, 8 up, 8 in 2024-05-09T04:12:42.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: mgrmap e42: y(active, since 2m), standbys: x 2024-05-09T04:12:42.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: overall HEALTH_OK 2024-05-09T04:12:42.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:42.396 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:42.397 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:42 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:42.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: mon.b calling monitor election 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: mon.c calling monitor election 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: mon.a calling monitor election 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: fsmap 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: osdmap e112: 8 total, 8 up, 8 in 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: mgrmap e42: y(active, since 2m), standbys: x 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: overall HEALTH_OK 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: mon.b calling monitor election 2024-05-09T04:12:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: mon.c calling monitor election 2024-05-09T04:12:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: mon.a calling monitor election 2024-05-09T04:12:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-09T04:12:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} 2024-05-09T04:12:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: fsmap 2024-05-09T04:12:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: osdmap e112: 8 total, 8 up, 8 in 2024-05-09T04:12:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: mgrmap e42: y(active, since 2m), standbys: x 2024-05-09T04:12:42.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: overall HEALTH_OK 2024-05-09T04:12:42.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:42.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:42.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:42 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:44.526 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:44 smithi132 ceph-mon[66347]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:12:44.526 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:44 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:12:44.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:44 smithi078 ceph-mon[22905]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:12:44.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:44 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:12:44.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:44 smithi078 ceph-mon[27629]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:12:44.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:44 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:12:46.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:46 smithi078 ceph-mon[22905]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:46.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:46 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:46.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:46 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:46.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:12:46 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:12:46.058+0000 7fd12b5df700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-05-09T04:12:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:46 smithi078 ceph-mon[27629]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:46 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:46 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:46.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:46 smithi132 ceph-mon[66347]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:46.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:46 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:46.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:46 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:47.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:47 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:47.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:47 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:47.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:47 smithi132 ceph-mon[66347]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:48.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:47 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:48.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:47 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:48.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:47 smithi078 ceph-mon[22905]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:48.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:47 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:48.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:47 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:48.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:47 smithi078 ceph-mon[27629]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:49.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:12:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:12:49] "GET /metrics HTTP/1.1" 200 38014 "" "Prometheus/2.43.0" 2024-05-09T04:12:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:12:50.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:12:50.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:50.508 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:50.509 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:50 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:50.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-05-09T04:12:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:12:50.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:12:50.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:12:50.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:12:50.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:12:50.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:12:50.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:12:50.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:12:50.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:12:50.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:12:50.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:12:50.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:12:50.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:12:50.517 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:12:50.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:12:50.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-09T04:12:50.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:12:50.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:12:50.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:12:50.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:12:50.520 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:50.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:50 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:51.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Detected new or changed devices on smithi132 2024-05-09T04:12:51.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all crash 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mds 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all nfs 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all node-exporter 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all prometheus 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all alertmanager 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all grafana 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all loki 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all promtail 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Finalizing container_image settings 2024-05-09T04:12:51.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: Upgrade: Complete! 2024-05-09T04:12:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:51.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:51 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:51.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Detected new or changed devices on smithi132 2024-05-09T04:12:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all crash 2024-05-09T04:12:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all mds 2024-05-09T04:12:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:12:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-09T04:12:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:12:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all nfs 2024-05-09T04:12:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:12:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all node-exporter 2024-05-09T04:12:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all prometheus 2024-05-09T04:12:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all alertmanager 2024-05-09T04:12:51.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all grafana 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all loki 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all promtail 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Finalizing container_image settings 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: Upgrade: Complete! 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Detected new or changed devices on smithi132 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all crash 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all mds 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-09T04:12:51.513 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all nfs 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all node-exporter 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all prometheus 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all alertmanager 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all grafana 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all loki 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all promtail 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Finalizing container_image settings 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: Upgrade: Complete! 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:12:51.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:51 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:52.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:52 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:52.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:52 smithi132 ceph-mon[66347]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:52.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:52 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:52.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:52 smithi078 ceph-mon[22905]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:52.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:52 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:12:52.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:52 smithi078 ceph-mon[27629]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:54.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:54 smithi132 ceph-mon[66347]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:54.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:54 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:54.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:54 smithi078 ceph-mon[22905]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:54.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:54 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:54.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:54 smithi078 ceph-mon[27629]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:54.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:54 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:12:56.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:56 smithi078 ceph-mon[22905]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:56.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:56 smithi078 ceph-mon[27629]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:56.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:56 smithi132 ceph-mon[66347]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:12:58.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:58 smithi078 ceph-mon[22905]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:58.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:58 smithi078 ceph-mon[27629]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:58.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:58 smithi132 ceph-mon[66347]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:12:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:12:59 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:12:59.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:12:59 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:12:59.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:12:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:12:59] "GET /metrics HTTP/1.1" 200 38086 "" "Prometheus/2.43.0" 2024-05-09T04:12:59.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:12:59 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:13:00.594 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:00 smithi078 ceph-mon[22905]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:00.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:00 smithi078 ceph-mon[27629]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:00.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:00 smithi132 ceph-mon[66347]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:02.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:02 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:02.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:02 smithi132 ceph-mon[66347]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:02.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:02 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:02.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:02 smithi078 ceph-mon[27629]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:02.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:02 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:02.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:02 smithi078 ceph-mon[22905]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:04.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:04 smithi132 ceph-mon[66347]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:04.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:04 smithi078 ceph-mon[27629]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:04.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:04 smithi078 ceph-mon[22905]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:06.556 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:06 smithi132 ceph-mon[66347]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:06.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:06 smithi078 ceph-mon[27629]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:06.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:06 smithi078 ceph-mon[22905]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:08.697 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:08 smithi078 ceph-mon[27629]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:08.698 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:08 smithi078 ceph-mon[22905]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:08.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:08 smithi132 ceph-mon[66347]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:09.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T04:13:09.359 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:13:09] "GET /metrics HTTP/1.1" 200 38086 "" "Prometheus/2.43.0" 2024-05-09T04:13:10.557 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:10 smithi078 ceph-mon[22905]: from='client.15756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:10.557 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:10 smithi078 ceph-mon[22905]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:10.557 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:10 smithi078 ceph-mon[27629]: from='client.15756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:10.557 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:10 smithi078 ceph-mon[27629]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:10.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:10 smithi132 ceph-mon[66347]: from='client.15756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:10.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:10 smithi132 ceph-mon[66347]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:11.010 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:13:11.010 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (103s) 98s ago 23m 16.4M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:13:11.010 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (80s) 25s ago 23m 76.3M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:13:11.010 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (3m) 98s ago 22m 52.7M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:13:11.010 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (2m) 25s ago 28m 466M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:13:11.010 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (13m) 98s ago 30m 518M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:13:11.010 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (30m) 98s ago 30m 78.2M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T04:13:11.010 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (30s) 25s ago 28m 19.1M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:13:11.010 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (28m) 98s ago 28m 60.6M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T04:13:11.010 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (2m) 98s ago 24m 8404k - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:13:11.010 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (2m) 25s ago 24m 16.9M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:13:11.010 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (28m) 98s ago 28m 54.7M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:13:11.011 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (27m) 98s ago 27m 58.7M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:13:11.011 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (27m) 98s ago 27m 51.0M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:13:11.011 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (26m) 98s ago 26m 54.1M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:13:11.011 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (26m) 25s ago 26m 58.1M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:13:11.011 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (26m) 25s ago 26m 52.6M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:13:11.011 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (25m) 25s ago 25m 53.5M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:13:11.011 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (25m) 25s ago 25m 57.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:13:11.011 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (2m) 25s ago 24m 65.3M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:13:11.011 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (23m) 98s ago 23m 78.1M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:13:11.011 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (23m) 25s ago 23m 80.2M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:13:11.613 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-05-09T04:13:12.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:12 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:12.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:12 smithi078 ceph-mon[22905]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:12.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:12 smithi078 ceph-mon[22905]: from='client.15762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:12.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:12 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:12.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:12 smithi078 ceph-mon[27629]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:12.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:12 smithi078 ceph-mon[27629]: from='client.15762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:12.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:12 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:12.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:12 smithi132 ceph-mon[66347]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:12.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:12 smithi132 ceph-mon[66347]: from='client.15762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:13.671 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:13:14.396 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:14 smithi078 ceph-mon[22905]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:14.396 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:14 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/3494824202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:13:14.396 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:14 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:13:14.396 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:14 smithi078 ceph-mon[27629]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:14.396 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:14 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/3494824202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:13:14.396 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:14 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:13:14.420 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade status' 2024-05-09T04:13:14.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:14 smithi132 ceph-mon[66347]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:14.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:14 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/3494824202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:13:14.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:14 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:13:16.490 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:13:16.491 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": null, 2024-05-09T04:13:16.491 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": false, 2024-05-09T04:13:16.491 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "", 2024-05-09T04:13:16.491 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:13:16.491 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": null, 2024-05-09T04:13:16.491 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-05-09T04:13:16.491 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:13:16.491 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:13:16.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:16 smithi132 ceph-mon[66347]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:16 smithi078 ceph-mon[22905]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:16 smithi078 ceph-mon[27629]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:17.085 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph health detail' 2024-05-09T04:13:18.571 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:18 smithi078 ceph-mon[27629]: from='client.15774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:18.571 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:18 smithi078 ceph-mon[27629]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:18.572 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:18 smithi078 ceph-mon[22905]: from='client.15774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:18.572 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:18 smithi078 ceph-mon[22905]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:18.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:18 smithi132 ceph-mon[66347]: from='client.15774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:18.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:18 smithi132 ceph-mon[66347]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:19.098 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-05-09T04:13:19.486 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:19 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/2935735168' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:13:19.487 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:13:19] "GET /metrics HTTP/1.1" 200 38080 "" "Prometheus/2.43.0" 2024-05-09T04:13:19.487 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:19 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/2935735168' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:13:19.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:19 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2935735168' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:13:19.758 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-05-09T04:13:20.389 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:20 smithi078 ceph-mon[27629]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:20.389 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:20 smithi078 ceph-mon[22905]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:20.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:20 smithi132 ceph-mon[66347]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:22.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:22 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:22.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:22 smithi132 ceph-mon[66347]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:22.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:22 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:22.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:22 smithi078 ceph-mon[27629]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:22.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:22 smithi078 ceph-mon[22905]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:22.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:22 smithi078 ceph-mon[22905]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:23.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:23 smithi132 ceph-mon[66347]: from='client.15786 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:23.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:23 smithi132 ceph-mon[66347]: from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "mon", "hosts": "smithi078", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:23.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:23 smithi078 ceph-mon[22905]: from='client.15786 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:23.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:23 smithi078 ceph-mon[22905]: from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "mon", "hosts": "smithi078", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:23.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:23 smithi078 ceph-mon[27629]: from='client.15786 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:23.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:23 smithi078 ceph-mon[27629]: from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "mon", "hosts": "smithi078", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:24.106 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:13:24.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:24 smithi132 ceph-mon[66347]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:24.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:24 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:24.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:24 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:13:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:24 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:13:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:24 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:13:24.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:24 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:24.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:24 smithi078 ceph-mon[22905]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:24.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:24 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:24.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:24 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:13:24.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:24 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:13:24.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:24 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:13:24.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:24 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:24 smithi078 ceph-mon[27629]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:24 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:24 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:13:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:24 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:13:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:24 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:13:24.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:24 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:24.796 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-09T04:13:25.441 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:25 smithi078 ceph-mon[27629]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:13:25.441 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:25 smithi078 ceph-mon[27629]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:13:25.441 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:25 smithi078 ceph-mon[22905]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:13:25.441 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:25 smithi078 ceph-mon[22905]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:13:25.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:25 smithi132 ceph-mon[66347]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:13:25.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:25 smithi132 ceph-mon[66347]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:13:26.574 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:26 smithi078 ceph-mon[22905]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:26.574 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:26 smithi078 ceph-mon[27629]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:26.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:26 smithi132 ceph-mon[66347]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:28.059 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:13:28.318 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[27629]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:28.318 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:28.318 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[27629]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:13:28.318 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[27629]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:13:28.319 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:13:28.319 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:13:28.319 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[27629]: Upgrade: Setting container_image for all mgr 2024-05-09T04:13:28.319 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:28.319 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-09T04:13:28.319 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-05-09T04:13:28.319 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[27629]: Upgrade: It appears safe to stop mon.a 2024-05-09T04:13:28.319 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[22905]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:28.319 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:28.319 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[22905]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:13:28.319 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[22905]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:13:28.319 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:13:28.320 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:13:28.320 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[22905]: Upgrade: Setting container_image for all mgr 2024-05-09T04:13:28.320 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:28.320 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-09T04:13:28.320 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-05-09T04:13:28.320 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:28 smithi078 ceph-mon[22905]: Upgrade: It appears safe to stop mon.a 2024-05-09T04:13:28.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:28 smithi132 ceph-mon[66347]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:28.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:28 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:28 smithi132 ceph-mon[66347]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:13:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:28 smithi132 ceph-mon[66347]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:13:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:28 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:13:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:28 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:13:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:28 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mgr 2024-05-09T04:13:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:28 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:28 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-09T04:13:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:28 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-05-09T04:13:28.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:28 smithi132 ceph-mon[66347]: Upgrade: It appears safe to stop mon.a 2024-05-09T04:13:28.755 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:13:28.755 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (2m) 116s ago 24m 16.4M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:13:28.755 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (98s) 43s ago 23m 76.3M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:13:28.755 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (3m) 116s ago 23m 52.7M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:13:28.755 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (3m) 43s ago 28m 466M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:13:28.755 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (13m) 116s ago 30m 518M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:13:28.755 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (30m) 116s ago 30m 78.2M 2048M 17.2.0 e1d6a67b021e de97a71c7cb5 2024-05-09T04:13:28.755 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (48s) 43s ago 28m 19.1M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:13:28.755 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (29m) 116s ago 29m 60.6M 2048M 17.2.0 e1d6a67b021e 7563b8437b55 2024-05-09T04:13:28.755 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (2m) 116s ago 24m 8404k - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:13:28.755 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (2m) 43s ago 24m 16.9M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:13:28.755 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (28m) 116s ago 28m 54.7M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:13:28.756 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (27m) 116s ago 27m 58.7M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:13:28.756 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (27m) 116s ago 27m 51.0M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:13:28.756 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (27m) 116s ago 27m 54.1M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:13:28.756 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (26m) 43s ago 26m 58.1M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:13:28.756 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (26m) 43s ago 26m 52.6M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:13:28.756 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (26m) 43s ago 26m 53.5M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:13:28.756 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (25m) 43s ago 25m 57.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:13:28.756 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (2m) 43s ago 24m 65.3M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:13:28.756 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (23m) 116s ago 23m 78.1M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:13:28.756 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (23m) 43s ago 23m 80.2M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:13:29.170 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:13:29.170 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:13:29.170 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-09T04:13:29.170 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 1 2024-05-09T04:13:29.170 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": {}, 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:13:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:13:29.172 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:13:29.260 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:13:29] "GET /metrics HTTP/1.1" 200 38084 "" "Prometheus/2.43.0" 2024-05-09T04:13:29.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[22905]: from='client.25699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:29.589 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:13:29.589 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:29.589 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:13:29.589 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:13:29.589 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[22905]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:13:29.589 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[22905]: from='client.? 172.21.15.78:0/20203439' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:13:29.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[27629]: from='client.25699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:29.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:13:29.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:29.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:13:29.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:13:29.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:13:29.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:29 smithi078 ceph-mon[27629]: from='client.? 172.21.15.78:0/20203439' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:13:29.591 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:13:29.591 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:13:29.591 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:13:29.591 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi078", 2024-05-09T04:13:29.591 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:13:29.591 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "0/2 daemons upgraded", 2024-05-09T04:13:29.591 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mon daemons", 2024-05-09T04:13:29.591 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:13:29.591 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:13:29.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:29 smithi132 ceph-mon[66347]: from='client.25699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:29.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:29 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:13:29.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:29 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' 2024-05-09T04:13:29.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:29 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:13:29.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:29 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:13:29.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:29 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:13:29.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:29 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/20203439' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:13:30.539 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:30 smithi078 ceph-mon[27629]: from='client.15804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:30.540 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:30 smithi078 ceph-mon[27629]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:30.540 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:30 smithi078 ceph-mon[27629]: from='client.25711 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:30.540 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:30 smithi078 ceph-mon[27629]: Upgrade: Updating mon.a 2024-05-09T04:13:30.540 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:30 smithi078 ceph-mon[27629]: Deploying daemon mon.a on smithi078 2024-05-09T04:13:30.540 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:30 smithi078 ceph-mon[22905]: from='client.15804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:30.540 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:30 smithi078 ceph-mon[22905]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:30.540 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:30 smithi078 ceph-mon[22905]: from='client.25711 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:30.540 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:30 smithi078 ceph-mon[22905]: Upgrade: Updating mon.a 2024-05-09T04:13:30.540 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:30 smithi078 ceph-mon[22905]: Deploying daemon mon.a on smithi078 2024-05-09T04:13:30.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:30 smithi132 ceph-mon[66347]: from='client.15804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:30 smithi132 ceph-mon[66347]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:30 smithi132 ceph-mon[66347]: from='client.25711 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:13:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:30 smithi132 ceph-mon[66347]: Upgrade: Updating mon.a 2024-05-09T04:13:30.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:30 smithi132 ceph-mon[66347]: Deploying daemon mon.a on smithi078 2024-05-09T04:13:31.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:30 smithi078 systemd[1]: Stopping Ceph mon.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:13:31.469 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:31 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a[22901]: 2024-05-09T04:13:31.085+0000 7f7f4fb4c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:13:31.469 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:31 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a[22901]: 2024-05-09T04:13:31.085+0000 7f7f4fb4c700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-05-09T04:13:31.469 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:31 smithi078 podman[82997]: 2024-05-09 04:13:31.208499814 +0000 UTC m=+0.238578524 container died de97a71c7cb50369d585b52c272637b11e40f5dd85f89fb697ce1dcef897d9ee (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, version=8, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, vendor=Red Hat, Inc., release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, GIT_BRANCH=HEAD, ceph=True) 2024-05-09T04:13:31.470 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:31 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:13:31] ENGINE Bus STOPPING 2024-05-09T04:13:31.470 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:31 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:13:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-09T04:13:31.470 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:31 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:13:31] ENGINE Bus STOPPED 2024-05-09T04:13:31.470 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:31 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:13:31] ENGINE Bus STARTING 2024-05-09T04:13:31.470 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:31 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:13:31] ENGINE Serving on http://:::9283 2024-05-09T04:13:31.470 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:31 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:13:31] ENGINE Bus STARTED 2024-05-09T04:13:31.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:31 smithi078 podman[82997]: 2024-05-09 04:13:31.577316149 +0000 UTC m=+0.607394860 container cleanup de97a71c7cb50369d585b52c272637b11e40f5dd85f89fb697ce1dcef897d9ee (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vcs-type=git, version=8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vendor=Red Hat, Inc., name=centos-stream, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, com.redhat.component=centos-stream-container) 2024-05-09T04:13:31.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:31 smithi078 bash[82997]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a 2024-05-09T04:13:32.208 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:31 smithi078 podman[83022]: 2024-05-09 04:13:31.862349106 +0000 UTC m=+0.651218707 container remove de97a71c7cb50369d585b52c272637b11e40f5dd85f89fb697ce1dcef897d9ee (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, distribution-scope=public, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, io.openshift.expose-services=, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, version=8, name=centos-stream, maintainer=Guillaume Abrioux , release=754, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8) 2024-05-09T04:13:32.208 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.a.service: Deactivated successfully. 2024-05-09T04:13:32.208 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 systemd[1]: Stopped Ceph mon.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:13:32.208 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.a.service: Consumed 25.875s CPU time. 2024-05-09T04:13:32.504 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 systemd[1]: Starting Ceph mon.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:13:32.504 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 podman[83158]: 2024-05-09 04:13:32.421904648 +0000 UTC m=+0.112328366 container create ffed80c876d1e5408ef0005eb5403fa26f98884cf2da7f164ca5849d4f8249c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:13:32.505 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 podman[83158]: 2024-05-09 04:13:32.327885183 +0000 UTC m=+0.018308916 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:13:32.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 podman[83158]: 2024-05-09 04:13:32.599647214 +0000 UTC m=+0.290070924 container init ffed80c876d1e5408ef0005eb5403fa26f98884cf2da7f164ca5849d4f8249c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504) 2024-05-09T04:13:32.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 podman[83158]: 2024-05-09 04:13:32.602892372 +0000 UTC m=+0.293316084 container start ffed80c876d1e5408ef0005eb5403fa26f98884cf2da7f164ca5849d4f8249c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-05-09T04:13:32.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: set uid:gid to 167:167 (ceph:ceph) 2024-05-09T04:13:32.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev), process ceph-mon, pid 2 2024-05-09T04:13:32.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: pidfile_write: ignore empty --pid-file 2024-05-09T04:13:32.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: load: jerasure load: lrc 2024-05-09T04:13:32.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: RocksDB version: 7.9.2 2024-05-09T04:13:32.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Git sha 0 2024-05-09T04:13:32.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Compile date 2024-05-08 23:53:35 2024-05-09T04:13:32.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: DB SUMMARY 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: DB Session ID: ICDZE8UEWV9Z275H9U45 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: CURRENT file: CURRENT 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: IDENTITY file: IDENTITY 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: MANIFEST file: MANIFEST-000015 size: 3079 Bytes 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000063.sst 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000061.log size: 98657 ; 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.error_if_exists: 0 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.create_if_missing: 0 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.paranoid_checks: 1 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.env: 0x55a4939bacc0 2024-05-09T04:13:32.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.fs: PosixFileSystem 2024-05-09T04:13:32.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.info_log: 0x55a494ad7dc0 2024-05-09T04:13:32.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_file_opening_threads: 16 2024-05-09T04:13:32.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.statistics: (nil) 2024-05-09T04:13:32.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.use_fsync: 0 2024-05-09T04:13:32.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_log_file_size: 0 2024-05-09T04:13:32.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-09T04:13:32.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-09T04:13:32.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.keep_log_file_num: 1000 2024-05-09T04:13:32.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.recycle_log_file_num: 0 2024-05-09T04:13:32.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.allow_fallocate: 1 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.allow_mmap_reads: 0 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.allow_mmap_writes: 0 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.use_direct_reads: 0 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.create_missing_column_families: 0 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.db_log_dir: 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.wal_dir: 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-09T04:13:32.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.advise_random_on_open: 1 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.db_write_buffer_size: 0 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.write_buffer_manager: 0x55a494aa9220 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.rate_limiter: (nil) 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.wal_recovery_mode: 2 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.enable_thread_tracking: 0 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.enable_pipelined_write: 0 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.unordered_write: 0 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-09T04:13:32.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.row_cache: None 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.wal_filter: None 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.allow_ingest_behind: 0 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.two_write_queues: 0 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.manual_wal_flush: 0 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.wal_compression: 0 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.atomic_flush: 0 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.log_readahead_size: 0 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.best_efforts_recovery: 0 2024-05-09T04:13:32.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.allow_data_in_errors: 0 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.db_host_id: __hostname__ 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_background_jobs: 2 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_background_compactions: -1 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_subcompactions: 1 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_total_wal_size: 0 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-09T04:13:32.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_open_files: -1 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bytes_per_sync: 0 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_readahead_size: 0 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_background_flushes: -1 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Compression algorithms supported: 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: kZSTD supported: 0 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: kXpressCompression supported: 0 2024-05-09T04:13:32.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: kLZ4HCCompression supported: 1 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: kZlibCompression supported: 1 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: kSnappyCompression supported: 1 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: kLZ4Compression supported: 1 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: kBZip2Compression supported: 0 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.merge_operator: 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_filter: None 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_filter_factory: None 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.sst_partitioner_factory: None 2024-05-09T04:13:32.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a494ad7f80) 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: index_type: 0 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: data_block_index_type: 0 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: index_shortening: 1 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: checksum: 4 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: no_block_cache: 0 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_cache: 0x55a494adb1f0 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-05-09T04:13:32.772 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_cache_options: 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: capacity : 536870912 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: num_shard_bits : 4 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: strict_capacity_limit : 0 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_cache_compressed: (nil) 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: persistent_cache: (nil) 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_size: 4096 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_size_deviation: 10 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_restart_interval: 16 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: index_block_restart_interval: 1 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: metadata_block_size: 4096 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: partition_filters: 0 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: use_delta_encoding: 1 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: filter_policy: bloomfilter 2024-05-09T04:13:32.773 INFO:journalctl@ceph.mon.a.smithi078.stdout: whole_key_filtering: 1 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout: verify_compression: 0 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout: format_version: 5 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout: enable_index_compression: 1 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout: block_align: 0 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout: max_auto_readahead_size: 262144 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout: prepopulate_block_cache: 0 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout: initial_auto_readahead_size: 8192 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout: num_file_reads_for_auto_readahead: 2 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.write_buffer_size: 33554432 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_write_buffer_number: 2 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compression: NoCompression 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bottommost_compression: Disabled 2024-05-09T04:13:32.774 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.prefix_extractor: nullptr 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.num_levels: 7 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-09T04:13:32.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compression_opts.level: 32767 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compression_opts.strategy: 0 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compression_opts.enabled: false 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.target_file_size_base: 67108864 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-09T04:13:32.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.arena_block_size: 1048576 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-09T04:13:32.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.disable_auto_compactions: 0 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.inplace_update_support: 0 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-09T04:13:32.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.bloom_locality: 0 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.max_successive_merges: 0 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.paranoid_file_checks: 0 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.force_consistency_checks: 1 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.report_bg_io_stats: 0 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.ttl: 2592000 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-09T04:13:32.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.enable_blob_files: false 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.min_blob_size: 0 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.blob_file_size: 268435456 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.blob_file_starting_level: 0 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 63.sst 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 65, last_sequence is 29287, log_number is 61,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 61 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 97f49480-b0d1-458b-be71-a7327b93913d 2024-05-09T04:13:32.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715228012632951, "job": 1, "event": "recovery_started", "wal_files": [61]} 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #61 mode 2 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715228012633758, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 66, "file_size": 98198, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 29281, "largest_seqno": 29413, "table_properties": {"data_size": 96583, "index_size": 321, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 2115, "raw_average_key_size": 26, "raw_value_size": 94838, "raw_average_value_size": 1170, "num_data_blocks": 14, "num_entries": 81, "num_filter_entries": 81, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1715228012, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "97f49480-b0d1-458b-be71-a7327b93913d", "db_session_id": "ICDZE8UEWV9Z275H9U45", "orig_file_number": 66, "seqno_to_time_mapping": "N/A"}} 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715228012633840, "job": 1, "event": "recovery_finished"} 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [db/version_set.cc:5047] Creating manifest 68 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000061.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a494bf8000 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: DB pointer 0x55a494be4000 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout: ** DB Stats ** 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-09T04:13:32.781 INFO:journalctl@ceph.mon.a.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: L0 1/0 95.90 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 184.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: L6 1/0 9.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: Sum 2/0 9.61 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 184.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 184.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T04:13:32.782 INFO:journalctl@ceph.mon.a.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 184.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: Cumulative compaction: 0.00 GB write, 31.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T04:13:32.783 INFO:journalctl@ceph.mon.a.smithi078.stdout: Interval compaction: 0.00 GB write, 31.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout: Block cache BinnedLRUCache@0x55a494adb1f0#2 capacity: 512.00 MB usage: 0.73 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 8e-06 secs_since: 0 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.31 KB,5.96046e-05%) IndexBlock(1,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: starting mon.a rank 0 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: mon.a@-1(???) e3 preinit fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: mon.a@-1(???).mds e1 new map 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: mon.a@-1(???).mds e1 print_map 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout: e1 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout: legacy client fscid: -1 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout: 2024-05-09T04:13:32.784 INFO:journalctl@ceph.mon.a.smithi078.stdout: No filesystems configured 2024-05-09T04:13:32.785 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: mon.a@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-09T04:13:32.785 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-05-09T04:13:32.785 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-05-09T04:13:32.785 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-05-09T04:13:32.785 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 ceph-mon[83176]: mon.a@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-05-09T04:13:32.785 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 bash[83158]: ffed80c876d1e5408ef0005eb5403fa26f98884cf2da7f164ca5849d4f8249c7 2024-05-09T04:13:32.785 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:32 smithi078 systemd[1]: Started Ceph mon.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:13:36.003 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:13:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: ignoring --setuser ceph since I am not root 2024-05-09T04:13:36.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:13:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: ignoring --setgroup ceph since I am not root 2024-05-09T04:13:36.004 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mgr[59236]: -- 172.21.15.132:0/2195705034 <== mon.2 v2:172.21.15.132:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a2a17225a0 con 0x55a2a16f7000 2024-05-09T04:13:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:13:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:13:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:13:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: mon.a calling monitor election 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: monmap epoch 3 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: last_changed 2024-05-09T03:44:32.387541+0000 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: created 2024-05-09T03:42:55.620159+0000 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: min_mon_release 17 (quincy) 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: election_strategy: 1 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.a 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: 1: [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] mon.c 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: 2: [v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0] mon.b 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: fsmap 2024-05-09T04:13:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: osdmap e112: 8 total, 8 up, 8 in 2024-05-09T04:13:36.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: mgrmap e42: y(active, since 3m), standbys: x 2024-05-09T04:13:36.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: overall HEALTH_OK 2024-05-09T04:13:36.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: from='mgr.25285 ' entity='' 2024-05-09T04:13:36.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:35 smithi132 ceph-mon[66347]: mgrmap e43: y(active, since 3m), standbys: x 2024-05-09T04:13:36.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ignoring --setuser ceph since I am not root 2024-05-09T04:13:36.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:35 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ignoring --setgroup ceph since I am not root 2024-05-09T04:13:36.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mgr[61747]: -- 172.21.15.78:0/313618440 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56262261a5a0 con 0x5626225eec00 2024-05-09T04:13:36.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: mon.a calling monitor election 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: monmap epoch 3 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: last_changed 2024-05-09T03:44:32.387541+0000 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: created 2024-05-09T03:42:55.620159+0000 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: min_mon_release 17 (quincy) 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: election_strategy: 1 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.a 2024-05-09T04:13:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: 1: [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] mon.c 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: 2: [v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0] mon.b 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: fsmap 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: osdmap e112: 8 total, 8 up, 8 in 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: mgrmap e42: y(active, since 3m), standbys: x 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: overall HEALTH_OK 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: from='mgr.25285 ' entity='' 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[27629]: mgrmap e43: y(active, since 3m), standbys: x 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: from='mgr.25285 172.21.15.78:0/2341707908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:13:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: mon.a calling monitor election 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: monmap epoch 3 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: last_changed 2024-05-09T03:44:32.387541+0000 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: created 2024-05-09T03:42:55.620159+0000 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: min_mon_release 17 (quincy) 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: election_strategy: 1 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.a 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: 1: [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] mon.c 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: 2: [v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0] mon.b 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: fsmap 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: osdmap e112: 8 total, 8 up, 8 in 2024-05-09T04:13:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: mgrmap e42: y(active, since 3m), standbys: x 2024-05-09T04:13:36.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: overall HEALTH_OK 2024-05-09T04:13:36.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: from='mgr.25285 ' entity='' 2024-05-09T04:13:36.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:35 smithi078 ceph-mon[83176]: mgrmap e43: y(active, since 3m), standbys: x 2024-05-09T04:13:38.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:37 smithi132 ceph-mon[66347]: mgrmap e44: y(active, since 3m), standbys: x 2024-05-09T04:13:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:37 smithi078 ceph-mon[83176]: mgrmap e44: y(active, since 3m), standbys: x 2024-05-09T04:13:38.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:37 smithi078 ceph-mon[27629]: mgrmap e44: y(active, since 3m), standbys: x 2024-05-09T04:13:43.253 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:13:42 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: [09/May/2024:04:13:42] ENGINE Bus STARTING 2024-05-09T04:13:43.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:13:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: CherryPy Checker: 2024-05-09T04:13:43.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:13:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: The Application mounted at '' has an empty config. 2024-05-09T04:13:43.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:13:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: 2024-05-09T04:13:43.254 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:13:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: [09/May/2024:04:13:43] ENGINE Serving on http://:::9283 2024-05-09T04:13:43.255 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:13:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x[59232]: [09/May/2024:04:13:43] ENGINE Bus STARTED 2024-05-09T04:13:43.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: Active manager daemon y restarted 2024-05-09T04:13:43.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: Activating manager daemon y 2024-05-09T04:13:43.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: osdmap e113: 8 total, 8 up, 8 in 2024-05-09T04:13:43.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: mgrmap e45: y(active, starting, since 0.00850855s), standbys: x 2024-05-09T04:13:43.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:13:43.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:13:43.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:13:43.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T04:13:43.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T04:13:43.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:13:43.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:13:43.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:13:43.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:13:43.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:13:43.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:13:43.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:13:43.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:13:43.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T04:13:43.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T04:13:43.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T04:13:43.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: Manager daemon y is now available 2024-05-09T04:13:43.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: Standby manager daemon x restarted 2024-05-09T04:13:43.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: Standby manager daemon x started 2024-05-09T04:13:43.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.? 172.21.15.132:0/701682883' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T04:13:43.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.? 172.21.15.132:0/701682883' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:13:43.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.? 172.21.15.132:0/701682883' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T04:13:43.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:42 smithi132 ceph-mon[66347]: from='mgr.? 172.21.15.132:0/701682883' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:13:43.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:42 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:13:42] ENGINE Bus STARTING 2024-05-09T04:13:43.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: CherryPy Checker: 2024-05-09T04:13:43.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: The Application mounted at '' has an empty config. 2024-05-09T04:13:43.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:13:43.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:13:43] ENGINE Serving on http://:::9283 2024-05-09T04:13:43.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:13:43] ENGINE Bus STARTED 2024-05-09T04:13:43.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: Active manager daemon y restarted 2024-05-09T04:13:43.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: Activating manager daemon y 2024-05-09T04:13:43.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: osdmap e113: 8 total, 8 up, 8 in 2024-05-09T04:13:43.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: mgrmap e45: y(active, starting, since 0.00850855s), standbys: x 2024-05-09T04:13:43.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:13:43.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:13:43.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:13:43.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T04:13:43.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T04:13:43.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:13:43.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:13:43.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:13:43.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:13:43.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:13:43.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:13:43.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:13:43.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:13:43.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T04:13:43.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T04:13:43.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T04:13:43.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: Manager daemon y is now available 2024-05-09T04:13:43.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: Standby manager daemon x restarted 2024-05-09T04:13:43.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: Standby manager daemon x started 2024-05-09T04:13:43.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/701682883' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T04:13:43.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/701682883' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:13:43.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/701682883' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T04:13:43.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[27629]: from='mgr.? 172.21.15.132:0/701682883' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:13:43.267 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: Active manager daemon y restarted 2024-05-09T04:13:43.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: Activating manager daemon y 2024-05-09T04:13:43.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: osdmap e113: 8 total, 8 up, 8 in 2024-05-09T04:13:43.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: mgrmap e45: y(active, starting, since 0.00850855s), standbys: x 2024-05-09T04:13:43.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:13:43.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:13:43.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:13:43.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-09T04:13:43.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-09T04:13:43.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:13:43.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:13:43.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:13:43.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:13:43.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:13:43.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:13:43.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:13:43.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:13:43.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-09T04:13:43.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-09T04:13:43.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-09T04:13:43.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: Manager daemon y is now available 2024-05-09T04:13:43.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: Standby manager daemon x restarted 2024-05-09T04:13:43.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: Standby manager daemon x started 2024-05-09T04:13:43.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.? 172.21.15.132:0/701682883' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-09T04:13:43.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.? 172.21.15.132:0/701682883' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-09T04:13:43.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.? 172.21.15.132:0/701682883' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-09T04:13:43.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:42 smithi078 ceph-mon[83176]: from='mgr.? 172.21.15.132:0/701682883' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-09T04:13:44.230 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:13:44.230 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:13:44.230 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:13:44.230 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T04:13:44.230 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:43 smithi132 ceph-mon[66347]: [09/May/2024:04:13:43] ENGINE Bus STARTING 2024-05-09T04:13:44.230 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:43 smithi132 ceph-mon[66347]: [09/May/2024:04:13:43] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T04:13:44.231 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:43 smithi132 ceph-mon[66347]: [09/May/2024:04:13:43] ENGINE Serving on http://172.21.15.78:8765 2024-05-09T04:13:44.231 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:43 smithi132 ceph-mon[66347]: [09/May/2024:04:13:43] ENGINE Bus STARTED 2024-05-09T04:13:44.231 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:43 smithi132 ceph-mon[66347]: mgrmap e46: y(active, since 1.01098s), standbys: x 2024-05-09T04:13:44.246 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:13:43.881+0000 7f81013d6700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-05-09T04:13:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:13:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:13:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:13:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T04:13:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[27629]: [09/May/2024:04:13:43] ENGINE Bus STARTING 2024-05-09T04:13:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[27629]: [09/May/2024:04:13:43] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T04:13:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[27629]: [09/May/2024:04:13:43] ENGINE Serving on http://172.21.15.78:8765 2024-05-09T04:13:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[27629]: [09/May/2024:04:13:43] ENGINE Bus STARTED 2024-05-09T04:13:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[27629]: mgrmap e46: y(active, since 1.01098s), standbys: x 2024-05-09T04:13:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:13:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:13:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-09T04:13:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-09T04:13:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[83176]: [09/May/2024:04:13:43] ENGINE Bus STARTING 2024-05-09T04:13:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[83176]: [09/May/2024:04:13:43] ENGINE Serving on https://172.21.15.78:7150 2024-05-09T04:13:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[83176]: [09/May/2024:04:13:43] ENGINE Serving on http://172.21.15.78:8765 2024-05-09T04:13:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[83176]: [09/May/2024:04:13:43] ENGINE Bus STARTED 2024-05-09T04:13:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:43 smithi078 ceph-mon[83176]: mgrmap e46: y(active, since 1.01098s), standbys: x 2024-05-09T04:13:45.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:44 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:45.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:44 smithi132 ceph-mon[66347]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:13:45.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:44 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:45.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:44 smithi078 ceph-mon[27629]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:13:45.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:44 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:45.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:44 smithi078 ceph-mon[83176]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:13:46.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:45 smithi132 ceph-mon[66347]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:13:46.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:45 smithi132 ceph-mon[66347]: mgrmap e47: y(active, since 2s), standbys: x 2024-05-09T04:13:46.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:45 smithi078 ceph-mon[83176]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:13:46.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:45 smithi078 ceph-mon[83176]: mgrmap e47: y(active, since 2s), standbys: x 2024-05-09T04:13:46.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:45 smithi078 ceph-mon[27629]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:13:46.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:45 smithi078 ceph-mon[27629]: mgrmap e47: y(active, since 2s), standbys: x 2024-05-09T04:13:48.248 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:47 smithi132 ceph-mon[66347]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:13:48.248 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:47 smithi132 ceph-mon[66347]: mgrmap e48: y(active, since 4s), standbys: x 2024-05-09T04:13:48.248 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:48.248 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:48.248 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:48.248 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:47 smithi078 ceph-mon[27629]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:13:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:47 smithi078 ceph-mon[27629]: mgrmap e48: y(active, since 4s), standbys: x 2024-05-09T04:13:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:47 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:47 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:47 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:47 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:48.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:47 smithi078 ceph-mon[83176]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:13:48.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:47 smithi078 ceph-mon[83176]: mgrmap e48: y(active, since 4s), standbys: x 2024-05-09T04:13:48.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:48.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:48.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:48.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:49.368 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:13:49] "GET /metrics HTTP/1.1" 200 35289 "" "Prometheus/2.43.0" 2024-05-09T04:13:49.747 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:49 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:49.747 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:49 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:49.747 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:49 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:49.748 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:49 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:49.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:49 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:49.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:49 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:49.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:49 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:49.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:49 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:49.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:49 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:49.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:49 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:49.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:49 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:49.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:49 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:50.720 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:50 smithi132 ceph-mon[66347]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:13:50.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:50 smithi078 ceph-mon[83176]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:13:50.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:50 smithi078 ceph-mon[27629]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:13:53.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:52 smithi132 ceph-mon[66347]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T04:13:53.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:52 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:52 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:52 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:52 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:52 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:13:53.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:52 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:13:53.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[83176]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T04:13:53.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:13:53.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:13:53.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[27629]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-09T04:13:53.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:13:53.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:52 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:13:53.902 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[83176]: Detected new or changed devices on smithi078 2024-05-09T04:13:53.902 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[83176]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T04:13:53.902 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[83176]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T04:13:53.902 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[83176]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:13:53.902 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[83176]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:13:53.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[27629]: Detected new or changed devices on smithi078 2024-05-09T04:13:53.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T04:13:53.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T04:13:53.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[27629]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:13:53.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[27629]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:13:53.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.904 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.904 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:53.904 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:53 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:54.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:53 smithi132 ceph-mon[66347]: Detected new or changed devices on smithi078 2024-05-09T04:13:54.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:53 smithi132 ceph-mon[66347]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-09T04:13:54.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:53 smithi132 ceph-mon[66347]: Updating smithi132:/etc/ceph/ceph.conf 2024-05-09T04:13:54.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:53 smithi132 ceph-mon[66347]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:13:54.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:53 smithi132 ceph-mon[66347]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.conf 2024-05-09T04:13:54.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:53 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:54.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:53 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:54.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:53 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:54.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:53 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:54.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:53 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:54.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[27629]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T04:13:54.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[27629]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:13:54.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[27629]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:13:54.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[27629]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:13:54.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[27629]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:13:54.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:13:54.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:13:54.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-09T04:13:54.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-05-09T04:13:54.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[83176]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T04:13:54.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[83176]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:13:54.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[83176]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:13:54.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[83176]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:13:54.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[83176]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:13:54.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:13:54.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:13:54.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-09T04:13:54.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-05-09T04:13:55.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:54 smithi132 ceph-mon[66347]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-09T04:13:55.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:54 smithi132 ceph-mon[66347]: Updating smithi132:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:13:55.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:54 smithi132 ceph-mon[66347]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-09T04:13:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:54 smithi132 ceph-mon[66347]: Updating smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:13:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:54 smithi132 ceph-mon[66347]: Updating smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/config/ceph.client.admin.keyring 2024-05-09T04:13:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:13:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:13:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-09T04:13:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-05-09T04:13:55.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:55 smithi078 ceph-mon[27629]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:55.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:55 smithi078 ceph-mon[27629]: Upgrade: It appears safe to stop mon.c 2024-05-09T04:13:55.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:55 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:55.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:55 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:13:55.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:55 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:13:55.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:55 smithi078 ceph-mon[27629]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:13:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:55 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:55 smithi078 ceph-mon[83176]: Upgrade: It appears safe to stop mon.c 2024-05-09T04:13:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:55 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:55 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:13:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:55 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:13:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:55 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:13:56.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:55 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:13:56.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:55 smithi132 ceph-mon[66347]: Upgrade: It appears safe to stop mon.c 2024-05-09T04:13:56.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:55 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:13:56.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:55 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:13:56.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:55 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:13:56.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:55 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:13:56.950 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:56 smithi078 ceph-mon[27629]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:13:56.951 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:56 smithi078 ceph-mon[27629]: Upgrade: Updating mon.c 2024-05-09T04:13:56.951 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:56 smithi078 ceph-mon[27629]: Deploying daemon mon.c on smithi078 2024-05-09T04:13:56.951 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:56 smithi078 ceph-mon[83176]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:13:56.951 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:56 smithi078 ceph-mon[83176]: Upgrade: Updating mon.c 2024-05-09T04:13:56.951 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:13:56 smithi078 ceph-mon[83176]: Deploying daemon mon.c on smithi078 2024-05-09T04:13:57.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:56 smithi132 ceph-mon[66347]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:13:57.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:56 smithi132 ceph-mon[66347]: Upgrade: Updating mon.c 2024-05-09T04:13:57.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:13:56 smithi132 ceph-mon[66347]: Deploying daemon mon.c on smithi078 2024-05-09T04:13:57.488 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:57 smithi078 systemd[1]: Stopping Ceph mon.c for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:13:57.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:57 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c[27625]: 2024-05-09T04:13:57.487+0000 7ff0f02d3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:13:57.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:57 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c[27625]: 2024-05-09T04:13:57.487+0000 7ff0f02d3700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-05-09T04:13:57.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:57 smithi078 podman[88243]: 2024-05-09 04:13:57.529378174 +0000 UTC m=+0.150666094 container died 7563b8437b5529f1d4f1bb293790f9c3c375a5da74f7588b36c3e831d10964b9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, version=8, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, ceph=True, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vcs-type=git, architecture=x86_64, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870) 2024-05-09T04:13:58.156 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:57 smithi078 podman[88243]: 2024-05-09 04:13:57.879917202 +0000 UTC m=+0.501205122 container cleanup 7563b8437b5529f1d4f1bb293790f9c3c375a5da74f7588b36c3e831d10964b9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c, maintainer=Guillaume Abrioux , vcs-type=git, RELEASE=HEAD, release=754, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, version=8, GIT_BRANCH=HEAD, GIT_CLEAN=True, distribution-scope=public, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True) 2024-05-09T04:13:58.156 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:57 smithi078 bash[88243]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c 2024-05-09T04:13:58.505 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:58 smithi078 podman[88257]: 2024-05-09 04:13:58.156452823 +0000 UTC m=+0.625550109 container remove 7563b8437b5529f1d4f1bb293790f9c3c375a5da74f7588b36c3e831d10964b9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, release=754, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, name=centos-stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True) 2024-05-09T04:13:58.505 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:58 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.c.service: Deactivated successfully. 2024-05-09T04:13:58.505 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:58 smithi078 systemd[1]: Stopped Ceph mon.c for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:13:58.505 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:58 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.c.service: Consumed 15.616s CPU time. 2024-05-09T04:13:58.505 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:58 smithi078 systemd[1]: Starting Ceph mon.c for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:13:58.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:58 smithi078 podman[88388]: 2024-05-09 04:13:58.626611355 +0000 UTC m=+0.018407912 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:13:58.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:58 smithi078 podman[88388]: 2024-05-09 04:13:58.740612962 +0000 UTC m=+0.132409501 container create f45be4f14efdb450a0af95dc5ae16a010e4d46bacad28ee6279a56bb8c0ca865 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:13:59.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:13:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:13:59] "GET /metrics HTTP/1.1" 200 38091 "" "Prometheus/2.43.0" 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:58 smithi078 podman[88388]: 2024-05-09 04:13:58.985282231 +0000 UTC m=+0.377078780 container init f45be4f14efdb450a0af95dc5ae16a010e4d46bacad28ee6279a56bb8c0ca865 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS) 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:58 smithi078 podman[88388]: 2024-05-09 04:13:58.988720835 +0000 UTC m=+0.380517375 container start f45be4f14efdb450a0af95dc5ae16a010e4d46bacad28ee6279a56bb8c0ca865 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True) 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: set uid:gid to 167:167 (ceph:ceph) 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev), process ceph-mon, pid 2 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: pidfile_write: ignore empty --pid-file 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: load: jerasure load: lrc 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: RocksDB version: 7.9.2 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Git sha 0 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Compile date 2024-05-08 23:53:35 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: DB SUMMARY 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: DB Session ID: Q10WMULLVDHODPRHIC3J 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: CURRENT file: CURRENT 2024-05-09T04:13:59.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: IDENTITY file: IDENTITY 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3194 Bytes 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000060.sst 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000058.log size: 2545132 ; 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.error_if_exists: 0 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.create_if_missing: 0 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.paranoid_checks: 1 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.env: 0x562c0e3a4cc0 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.fs: PosixFileSystem 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.info_log: 0x562c0ed65dc0 2024-05-09T04:13:59.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_file_opening_threads: 16 2024-05-09T04:13:59.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.statistics: (nil) 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.use_fsync: 0 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_log_file_size: 0 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.keep_log_file_num: 1000 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.recycle_log_file_num: 0 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.allow_fallocate: 1 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.allow_mmap_reads: 0 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.allow_mmap_writes: 0 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.use_direct_reads: 0 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.create_missing_column_families: 0 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.db_log_dir: 2024-05-09T04:13:59.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.wal_dir: 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.advise_random_on_open: 1 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.db_write_buffer_size: 0 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.write_buffer_manager: 0x562c0ed37220 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.rate_limiter: (nil) 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-09T04:13:59.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.wal_recovery_mode: 2 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.enable_thread_tracking: 0 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.enable_pipelined_write: 0 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.unordered_write: 0 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.row_cache: None 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.wal_filter: None 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.allow_ingest_behind: 0 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.two_write_queues: 0 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.manual_wal_flush: 0 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.wal_compression: 0 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.atomic_flush: 0 2024-05-09T04:13:59.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.log_readahead_size: 0 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.best_efforts_recovery: 0 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.allow_data_in_errors: 0 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.db_host_id: __hostname__ 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_background_jobs: 2 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_background_compactions: -1 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_subcompactions: 1 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-09T04:13:59.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_total_wal_size: 0 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_open_files: -1 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bytes_per_sync: 0 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_readahead_size: 0 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_background_flushes: -1 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Compression algorithms supported: 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: kZSTD supported: 0 2024-05-09T04:13:59.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: kXpressCompression supported: 0 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: kLZ4HCCompression supported: 1 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: kZlibCompression supported: 1 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: kSnappyCompression supported: 1 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: kLZ4Compression supported: 1 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: kBZip2Compression supported: 0 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.merge_operator: 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_filter: None 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_filter_factory: None 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.sst_partitioner_factory: None 2024-05-09T04:13:59.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-09T04:13:59.271 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-09T04:13:59.271 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562c0ed65f80) 2024-05-09T04:13:59.271 INFO:journalctl@ceph.mon.c.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-05-09T04:13:59.271 INFO:journalctl@ceph.mon.c.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-09T04:13:59.271 INFO:journalctl@ceph.mon.c.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-09T04:13:59.271 INFO:journalctl@ceph.mon.c.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-05-09T04:13:59.271 INFO:journalctl@ceph.mon.c.smithi078.stdout: index_type: 0 2024-05-09T04:13:59.271 INFO:journalctl@ceph.mon.c.smithi078.stdout: data_block_index_type: 0 2024-05-09T04:13:59.271 INFO:journalctl@ceph.mon.c.smithi078.stdout: index_shortening: 1 2024-05-09T04:13:59.271 INFO:journalctl@ceph.mon.c.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-09T04:13:59.271 INFO:journalctl@ceph.mon.c.smithi078.stdout: checksum: 4 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: no_block_cache: 0 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_cache: 0x562c0ed691f0 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_cache_options: 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: capacity : 536870912 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: num_shard_bits : 4 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: strict_capacity_limit : 0 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_cache_compressed: (nil) 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: persistent_cache: (nil) 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_size: 4096 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_size_deviation: 10 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_restart_interval: 16 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: index_block_restart_interval: 1 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: metadata_block_size: 4096 2024-05-09T04:13:59.272 INFO:journalctl@ceph.mon.c.smithi078.stdout: partition_filters: 0 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout: use_delta_encoding: 1 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout: filter_policy: bloomfilter 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout: whole_key_filtering: 1 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout: verify_compression: 0 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout: format_version: 5 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout: enable_index_compression: 1 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout: block_align: 0 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout: max_auto_readahead_size: 262144 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout: prepopulate_block_cache: 0 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout: initial_auto_readahead_size: 8192 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout: num_file_reads_for_auto_readahead: 2 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.write_buffer_size: 33554432 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_write_buffer_number: 2 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compression: NoCompression 2024-05-09T04:13:59.273 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bottommost_compression: Disabled 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.prefix_extractor: nullptr 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.num_levels: 7 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-09T04:13:59.274 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compression_opts.level: 32767 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compression_opts.strategy: 0 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compression_opts.enabled: false 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.target_file_size_base: 67108864 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-09T04:13:59.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.arena_block_size: 1048576 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-09T04:13:59.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.disable_auto_compactions: 0 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.inplace_update_support: 0 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-09T04:13:59.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.bloom_locality: 0 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.max_successive_merges: 0 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.paranoid_file_checks: 0 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.force_consistency_checks: 1 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.report_bg_io_stats: 0 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.ttl: 2592000 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.enable_blob_files: false 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.min_blob_size: 0 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.blob_file_size: 268435456 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-09T04:13:59.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.blob_file_starting_level: 0 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 62, last_sequence is 32231, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: bec0f1ab-61ff-437e-a555-42fb2e7af8a0 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715228039018972, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715228039025358, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1609914, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 32233, "largest_seqno": 32658, "table_properties": {"data_size": 1606920, "index_size": 1442, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 4794, "raw_average_key_size": 23, "raw_value_size": 1602343, "raw_average_value_size": 8011, "num_data_blocks": 65, "num_entries": 200, "num_filter_entries": 200, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1715228039, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "bec0f1ab-61ff-437e-a555-42fb2e7af8a0", "db_session_id": "Q10WMULLVDHODPRHIC3J", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715228039025459, "job": 1, "event": "recovery_finished"} 2024-05-09T04:13:59.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x562c0ee86000 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: DB pointer 0x562c0ee72000 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout: ** DB Stats ** 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T04:13:59.280 INFO:journalctl@ceph.mon.c.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: L0 1/0 1.54 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 528.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: L6 1/0 10.05 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: Sum 2/0 11.58 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 528.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 528.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 528.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-09T04:13:59.281 INFO:journalctl@ceph.mon.c.smithi078.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout: Cumulative compaction: 0.00 GB write, 158.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout: Interval compaction: 0.00 GB write, 158.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout: Block cache BinnedLRUCache@0x562c0ed691f0#2 capacity: 512.00 MB usage: 2.17 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,1.58 KB,0.000301003%) Misc(1,0.00 KB,0%) 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: starting mon.c rank 1 at public addrs [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] at bind addrs [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: mon.c@-1(???) e3 preinit fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: mon.c@-1(???).mds e1 new map 2024-05-09T04:13:59.282 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: mon.c@-1(???).mds e1 print_map 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout: e1 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout: legacy client fscid: -1 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout: 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout: No filesystems configured 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: mon.c@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 ceph-mon[88406]: mon.c@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 bash[88388]: f45be4f14efdb450a0af95dc5ae16a010e4d46bacad28ee6279a56bb8c0ca865 2024-05-09T04:13:59.283 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:13:59 smithi078 systemd[1]: Started Ceph mon.c for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:13:59.886 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:14:00.552 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:14:00.552 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (2m) 13s ago 24m 18.8M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:14:00.552 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (2m) 12s ago 24m 80.8M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:14:00.552 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (3m) 13s ago 23m 58.2M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:14:00.552 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (3m) 12s ago 29m 463M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:14:00.552 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (13m) 13s ago 30m 490M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:14:00.552 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (28s) 13s ago 31m 31.2M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:14:00.552 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (79s) 12s ago 29m 36.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:14:00.552 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 starting - - - 2048M 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (3m) 13s ago 25m 8434k - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (3m) 12s ago 25m 17.3M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (28m) 13s ago 28m 54.8M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (28m) 13s ago 28m 58.7M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (28m) 13s ago 28m 51.2M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (27m) 13s ago 27m 54.2M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (27m) 12s ago 27m 58.2M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (26m) 12s ago 26m 52.7M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (26m) 12s ago 26m 53.5M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (26m) 12s ago 26m 57.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (2m) 12s ago 25m 65.5M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (23m) 13s ago 23m 78.9M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:14:00.553 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (23m) 12s ago 23m 80.6M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:14:00.989 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:14:00.989 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:14:00.990 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-09T04:14:00.991 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 5 2024-05-09T04:14:00.991 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:14:00.991 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:14:01.367 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:14:01.368 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:14:01.368 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:14:01.368 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi078", 2024-05-09T04:14:01.368 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:14:01.368 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/2 daemons upgraded", 2024-05-09T04:14:01.368 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mon daemons", 2024-05-09T04:14:01.368 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:14:01.368 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:14:01.705 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:01.705 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: from='client.44142 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:01.705 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: from='client.44148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:01.705 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: fsmap 2024-05-09T04:14:01.705 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: osdmap e113: 8 total, 8 up, 8 in 2024-05-09T04:14:01.705 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: mgrmap e48: y(active, since 17s), standbys: x 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: overall HEALTH_OK 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: mon.c calling monitor election 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: mon.b calling monitor election 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: mon.a calling monitor election 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: monmap epoch 4 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: last_changed 2024-05-09T04:14:00.592761+0000 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: created 2024-05-09T03:42:55.620159+0000 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: min_mon_release 19 (squid) 2024-05-09T04:14:01.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: election_strategy: 1 2024-05-09T04:14:01.707 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.a 2024-05-09T04:14:01.707 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: 1: [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] mon.c 2024-05-09T04:14:01.707 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: 2: [v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0] mon.b 2024-05-09T04:14:01.707 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: fsmap 2024-05-09T04:14:01.707 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: osdmap e113: 8 total, 8 up, 8 in 2024-05-09T04:14:01.707 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: mgrmap e48: y(active, since 17s), standbys: x 2024-05-09T04:14:01.707 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: overall HEALTH_OK 2024-05-09T04:14:01.707 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:01.707 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:01.707 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:14:01.707 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/3394589441' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:01.707 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: from='client.44142 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: from='client.44148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: fsmap 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: osdmap e113: 8 total, 8 up, 8 in 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: mgrmap e48: y(active, since 17s), standbys: x 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: overall HEALTH_OK 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: mon.c calling monitor election 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: mon.b calling monitor election 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: mon.a calling monitor election 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: monmap epoch 4 2024-05-09T04:14:01.708 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: last_changed 2024-05-09T04:14:00.592761+0000 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: created 2024-05-09T03:42:55.620159+0000 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: min_mon_release 19 (squid) 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: election_strategy: 1 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.a 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: 1: [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] mon.c 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: 2: [v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0] mon.b 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: fsmap 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: osdmap e113: 8 total, 8 up, 8 in 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: mgrmap e48: y(active, since 17s), standbys: x 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: overall HEALTH_OK 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:14:01.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:01 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/3394589441' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: from='client.44142 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: from='client.44148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: fsmap 2024-05-09T04:14:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: osdmap e113: 8 total, 8 up, 8 in 2024-05-09T04:14:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: mgrmap e48: y(active, since 17s), standbys: x 2024-05-09T04:14:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: overall HEALTH_OK 2024-05-09T04:14:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: mon.c calling monitor election 2024-05-09T04:14:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: mon.b calling monitor election 2024-05-09T04:14:02.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: mon.a calling monitor election 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: monmap epoch 4 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: fsid 2777c79e-0db6-11ef-bc97-c7b262605968 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: last_changed 2024-05-09T04:14:00.592761+0000 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: created 2024-05-09T03:42:55.620159+0000 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: min_mon_release 19 (squid) 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: election_strategy: 1 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.a 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: 1: [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] mon.c 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: 2: [v2:172.21.15.132:3300/0,v1:172.21.15.132:6789/0] mon.b 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: fsmap 2024-05-09T04:14:02.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: osdmap e113: 8 total, 8 up, 8 in 2024-05-09T04:14:02.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: mgrmap e48: y(active, since 17s), standbys: x 2024-05-09T04:14:02.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: overall HEALTH_OK 2024-05-09T04:14:02.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:02.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:02.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:14:02.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:01 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/3394589441' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:02.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:02 smithi078 ceph-mon[88406]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:14:02.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:02 smithi078 ceph-mon[88406]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:02.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:02 smithi078 ceph-mon[83176]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:14:02.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:02 smithi078 ceph-mon[83176]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:03.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:02 smithi132 ceph-mon[66347]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 7 op/s 2024-05-09T04:14:03.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:02 smithi132 ceph-mon[66347]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:04.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:14:04 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:14:04.034+0000 7f81013d6700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-05-09T04:14:05.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:04 smithi132 ceph-mon[66347]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:05.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:04 smithi078 ceph-mon[83176]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:05.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:04 smithi078 ceph-mon[88406]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:06.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:05 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:06.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:05 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:06.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:05 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:06.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:05 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:06.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:05 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:06.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:05 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:06.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:05 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:06.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:05 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:06.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:05 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:06.904 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:06 smithi132 ceph-mon[66347]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:06.904 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:06 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:06.904 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:06 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:06.990 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:06 smithi078 ceph-mon[83176]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:06.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:06 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:06.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:06 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:06.991 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:06 smithi078 ceph-mon[88406]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:06.991 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:06 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:06.991 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:06 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:09.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:08 smithi132 ceph-mon[66347]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:09.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:08 smithi078 ceph-mon[88406]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:09.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:08 smithi078 ceph-mon[83176]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:09.294 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:14:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:14:09] "GET /metrics HTTP/1.1" 200 38091 "" "Prometheus/2.43.0" 2024-05-09T04:14:10.725 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[88406]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:10.726 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:10.726 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:10.726 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:10.726 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:14:10.726 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:10.726 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[88406]: Reconfiguring mon.a (monmap changed)... 2024-05-09T04:14:10.726 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:14:10.726 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:14:10.726 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:10.726 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[88406]: Reconfiguring daemon mon.a on smithi078 2024-05-09T04:14:10.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[83176]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:10.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:10.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:10.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:10.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:14:10.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:10.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[83176]: Reconfiguring mon.a (monmap changed)... 2024-05-09T04:14:10.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:14:10.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:14:10.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:10.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:10 smithi078 ceph-mon[83176]: Reconfiguring daemon mon.a on smithi078 2024-05-09T04:14:10.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:10 smithi132 ceph-mon[66347]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:10.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:10 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:10.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:10 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:10.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:10 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:10.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:10 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:14:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:10 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:10 smithi132 ceph-mon[66347]: Reconfiguring mon.a (monmap changed)... 2024-05-09T04:14:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:10 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:14:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:10 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:14:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:10 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:10.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:10 smithi132 ceph-mon[66347]: Reconfiguring daemon mon.a on smithi078 2024-05-09T04:14:12.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[83176]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[83176]: Reconfiguring mgr.y (monmap changed)... 2024-05-09T04:14:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:14:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T04:14:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:12.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[83176]: Reconfiguring daemon mgr.y on smithi078 2024-05-09T04:14:12.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[88406]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:12.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:12.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:12.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[88406]: Reconfiguring mgr.y (monmap changed)... 2024-05-09T04:14:12.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:14:12.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T04:14:12.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:12.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:12 smithi078 ceph-mon[88406]: Reconfiguring daemon mgr.y on smithi078 2024-05-09T04:14:12.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:12 smithi132 ceph-mon[66347]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:12.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:12 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:12.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:12 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:12 smithi132 ceph-mon[66347]: Reconfiguring mgr.y (monmap changed)... 2024-05-09T04:14:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:12 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:14:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:12 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T04:14:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:12 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:12.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:12 smithi132 ceph-mon[66347]: Reconfiguring daemon mgr.y on smithi078 2024-05-09T04:14:13.923 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:13.923 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:13.923 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[83176]: Reconfiguring mon.c (monmap changed)... 2024-05-09T04:14:13.923 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:14:13.923 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:14:13.923 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:13.923 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[83176]: Reconfiguring daemon mon.c on smithi078 2024-05-09T04:14:13.923 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:14:13.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:13.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:13.924 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[88406]: Reconfiguring mon.c (monmap changed)... 2024-05-09T04:14:13.924 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:14:13.924 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:14:13.924 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:13.924 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[88406]: Reconfiguring daemon mon.c on smithi078 2024-05-09T04:14:13.924 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:14:14.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:13 smithi132 ceph-mon[66347]: Reconfiguring mon.c (monmap changed)... 2024-05-09T04:14:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:14:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:14:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:13 smithi132 ceph-mon[66347]: Reconfiguring daemon mon.c on smithi078 2024-05-09T04:14:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:14:14.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:14 smithi078 ceph-mon[88406]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:14.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:14 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:14.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:14 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:14.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:14 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-09T04:14:14.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:14 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:14.946 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:14 smithi078 ceph-mon[83176]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:14.946 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:14 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:14.946 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:14 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:14.946 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:14 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-09T04:14:14.946 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:14 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:14 smithi132 ceph-mon[66347]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:14 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:14 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:14 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-09T04:14:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:14 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:15.978 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:15 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:15.978 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:15 smithi078 ceph-mon[88406]: Reconfiguring osd.0 (monmap changed)... 2024-05-09T04:14:15.978 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:15 smithi078 ceph-mon[88406]: Reconfiguring daemon osd.0 on smithi078 2024-05-09T04:14:15.979 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:15 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:15.979 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:15 smithi078 ceph-mon[83176]: Reconfiguring osd.0 (monmap changed)... 2024-05-09T04:14:15.979 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:15 smithi078 ceph-mon[83176]: Reconfiguring daemon osd.0 on smithi078 2024-05-09T04:14:16.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:15 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:16.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:15 smithi132 ceph-mon[66347]: Reconfiguring osd.0 (monmap changed)... 2024-05-09T04:14:16.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:15 smithi132 ceph-mon[66347]: Reconfiguring daemon osd.0 on smithi078 2024-05-09T04:14:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:16 smithi132 ceph-mon[66347]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-09T04:14:17.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:17.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:16 smithi078 ceph-mon[88406]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:17.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:17.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:17.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-09T04:14:17.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:17.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:16 smithi078 ceph-mon[83176]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:17.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:17.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:17.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-09T04:14:17.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:18.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:17 smithi132 ceph-mon[66347]: Reconfiguring osd.1 (monmap changed)... 2024-05-09T04:14:18.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:17 smithi132 ceph-mon[66347]: Reconfiguring daemon osd.1 on smithi078 2024-05-09T04:14:18.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:17 smithi078 ceph-mon[83176]: Reconfiguring osd.1 (monmap changed)... 2024-05-09T04:14:18.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:17 smithi078 ceph-mon[83176]: Reconfiguring daemon osd.1 on smithi078 2024-05-09T04:14:18.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:17 smithi078 ceph-mon[88406]: Reconfiguring osd.1 (monmap changed)... 2024-05-09T04:14:18.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:17 smithi078 ceph-mon[88406]: Reconfiguring daemon osd.1 on smithi078 2024-05-09T04:14:19.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:18 smithi132 ceph-mon[66347]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:19.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:19.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-09T04:14:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:19.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:18 smithi078 ceph-mon[83176]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:19.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:19.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:19.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-09T04:14:19.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:19.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:18 smithi078 ceph-mon[88406]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:19.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:19.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:19.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-09T04:14:19.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:19.345 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:14:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:14:19] "GET /metrics HTTP/1.1" 200 38098 "" "Prometheus/2.43.0" 2024-05-09T04:14:20.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:19 smithi132 ceph-mon[66347]: Reconfiguring osd.2 (monmap changed)... 2024-05-09T04:14:20.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:19 smithi132 ceph-mon[66347]: Reconfiguring daemon osd.2 on smithi078 2024-05-09T04:14:20.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:19 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:20.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:19 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:20.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:19 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-09T04:14:20.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:19 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:20.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:19 smithi078 ceph-mon[83176]: Reconfiguring osd.2 (monmap changed)... 2024-05-09T04:14:20.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:19 smithi078 ceph-mon[83176]: Reconfiguring daemon osd.2 on smithi078 2024-05-09T04:14:20.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:19 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:20.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:19 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:20.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:19 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-09T04:14:20.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:19 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:20.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:19 smithi078 ceph-mon[88406]: Reconfiguring osd.2 (monmap changed)... 2024-05-09T04:14:20.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:19 smithi078 ceph-mon[88406]: Reconfiguring daemon osd.2 on smithi078 2024-05-09T04:14:20.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:19 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:20.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:19 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:20.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:19 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-09T04:14:20.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:19 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:21.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:20 smithi132 ceph-mon[66347]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:21.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:20 smithi132 ceph-mon[66347]: Reconfiguring osd.3 (monmap changed)... 2024-05-09T04:14:21.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:20 smithi132 ceph-mon[66347]: Reconfiguring daemon osd.3 on smithi078 2024-05-09T04:14:21.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:20 smithi078 ceph-mon[83176]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:21.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:20 smithi078 ceph-mon[83176]: Reconfiguring osd.3 (monmap changed)... 2024-05-09T04:14:21.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:20 smithi078 ceph-mon[83176]: Reconfiguring daemon osd.3 on smithi078 2024-05-09T04:14:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:20 smithi078 ceph-mon[88406]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:20 smithi078 ceph-mon[88406]: Reconfiguring osd.3 (monmap changed)... 2024-05-09T04:14:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:20 smithi078 ceph-mon[88406]: Reconfiguring daemon osd.3 on smithi078 2024-05-09T04:14:22.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[88406]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:22.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:22.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:22.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[88406]: Reconfiguring rgw.foo.smithi078.bfidtv (monmap changed)... 2024-05-09T04:14:22.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T04:14:22.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:22.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[88406]: Reconfiguring daemon rgw.foo.smithi078.bfidtv on smithi078 2024-05-09T04:14:22.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[83176]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:22.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:22.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:22.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[83176]: Reconfiguring rgw.foo.smithi078.bfidtv (monmap changed)... 2024-05-09T04:14:22.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T04:14:22.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:22.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:22 smithi078 ceph-mon[83176]: Reconfiguring daemon rgw.foo.smithi078.bfidtv on smithi078 2024-05-09T04:14:22.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:22 smithi132 ceph-mon[66347]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:22.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:22 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:22.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:22 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:22 smithi132 ceph-mon[66347]: Reconfiguring rgw.foo.smithi078.bfidtv (monmap changed)... 2024-05-09T04:14:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:22 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T04:14:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:22 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:22.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:22 smithi132 ceph-mon[66347]: Reconfiguring daemon rgw.foo.smithi078.bfidtv on smithi078 2024-05-09T04:14:24.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:23 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:24.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:23 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:24.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:23 smithi132 ceph-mon[66347]: Reconfiguring mon.b (monmap changed)... 2024-05-09T04:14:24.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:23 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:14:24.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:23 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:14:24.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:23 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:24.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:23 smithi132 ceph-mon[66347]: Reconfiguring daemon mon.b on smithi132 2024-05-09T04:14:24.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:23 smithi132 ceph-mon[66347]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:24.051 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:24.051 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:24.051 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[83176]: Reconfiguring mon.b (monmap changed)... 2024-05-09T04:14:24.051 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:14:24.051 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:14:24.051 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:24.051 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[83176]: Reconfiguring daemon mon.b on smithi132 2024-05-09T04:14:24.051 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[83176]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:24.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:24.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:24.052 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[88406]: Reconfiguring mon.b (monmap changed)... 2024-05-09T04:14:24.052 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-09T04:14:24.052 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-09T04:14:24.052 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:24.052 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[88406]: Reconfiguring daemon mon.b on smithi132 2024-05-09T04:14:24.052 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:23 smithi078 ceph-mon[88406]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:25.735 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:25 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:25.735 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:25.735 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:25.735 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:25 smithi132 ceph-mon[66347]: Reconfiguring mgr.x (monmap changed)... 2024-05-09T04:14:25.735 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:14:25.735 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T04:14:25.735 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:25.735 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:25 smithi132 ceph-mon[66347]: Reconfiguring daemon mgr.x on smithi132 2024-05-09T04:14:25.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[83176]: Reconfiguring mgr.x (monmap changed)... 2024-05-09T04:14:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:14:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T04:14:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:25.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[83176]: Reconfiguring daemon mgr.x on smithi132 2024-05-09T04:14:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:25.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:25.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[88406]: Reconfiguring mgr.x (monmap changed)... 2024-05-09T04:14:25.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-09T04:14:25.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-09T04:14:25.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:25.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:25 smithi078 ceph-mon[88406]: Reconfiguring daemon mgr.x on smithi132 2024-05-09T04:14:26.720 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:26 smithi132 ceph-mon[66347]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:26.720 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:26.720 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:26.720 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-09T04:14:26.720 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:26.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:26 smithi078 ceph-mon[88406]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:26.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-09T04:14:26.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:26 smithi078 ceph-mon[83176]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-09T04:14:26.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:27.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:27 smithi132 ceph-mon[66347]: Reconfiguring osd.4 (monmap changed)... 2024-05-09T04:14:27.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:27 smithi132 ceph-mon[66347]: Reconfiguring daemon osd.4 on smithi132 2024-05-09T04:14:27.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:27 smithi078 ceph-mon[83176]: Reconfiguring osd.4 (monmap changed)... 2024-05-09T04:14:27.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:27 smithi078 ceph-mon[83176]: Reconfiguring daemon osd.4 on smithi132 2024-05-09T04:14:27.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:27 smithi078 ceph-mon[88406]: Reconfiguring osd.4 (monmap changed)... 2024-05-09T04:14:27.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:27 smithi078 ceph-mon[88406]: Reconfiguring daemon osd.4 on smithi132 2024-05-09T04:14:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:28 smithi132 ceph-mon[66347]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:29.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-09T04:14:29.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:29.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:14:29.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:28 smithi078 ceph-mon[88406]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:29.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:29.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:29.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-09T04:14:29.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:29.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:14:29.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:28 smithi078 ceph-mon[83176]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:29.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:29.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:29.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-09T04:14:29.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:29.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:14:29.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:14:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:14:29] "GET /metrics HTTP/1.1" 200 38096 "" "Prometheus/2.43.0" 2024-05-09T04:14:30.028 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:29 smithi132 ceph-mon[66347]: Reconfiguring osd.5 (monmap changed)... 2024-05-09T04:14:30.028 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:29 smithi132 ceph-mon[66347]: Reconfiguring daemon osd.5 on smithi132 2024-05-09T04:14:30.028 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:29 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:30.028 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:29 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:30.028 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:29 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-09T04:14:30.028 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:29 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:30.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:29 smithi078 ceph-mon[88406]: Reconfiguring osd.5 (monmap changed)... 2024-05-09T04:14:30.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:29 smithi078 ceph-mon[88406]: Reconfiguring daemon osd.5 on smithi132 2024-05-09T04:14:30.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:29 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:30.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:29 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:30.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:29 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-09T04:14:30.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:29 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:30.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:29 smithi078 ceph-mon[83176]: Reconfiguring osd.5 (monmap changed)... 2024-05-09T04:14:30.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:29 smithi078 ceph-mon[83176]: Reconfiguring daemon osd.5 on smithi132 2024-05-09T04:14:30.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:29 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:30.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:29 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:30.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:29 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-09T04:14:30.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:29 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:31.030 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:30 smithi132 ceph-mon[66347]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:31.031 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:30 smithi132 ceph-mon[66347]: Reconfiguring osd.6 (monmap changed)... 2024-05-09T04:14:31.031 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:30 smithi132 ceph-mon[66347]: Reconfiguring daemon osd.6 on smithi132 2024-05-09T04:14:31.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:30 smithi078 ceph-mon[83176]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:31.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:30 smithi078 ceph-mon[83176]: Reconfiguring osd.6 (monmap changed)... 2024-05-09T04:14:31.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:30 smithi078 ceph-mon[83176]: Reconfiguring daemon osd.6 on smithi132 2024-05-09T04:14:31.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:30 smithi078 ceph-mon[88406]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:31.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:30 smithi078 ceph-mon[88406]: Reconfiguring osd.6 (monmap changed)... 2024-05-09T04:14:31.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:30 smithi078 ceph-mon[88406]: Reconfiguring daemon osd.6 on smithi132 2024-05-09T04:14:31.741 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:14:32.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:32 smithi132 ceph-mon[66347]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:32.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:32.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:32.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:32 smithi132 ceph-mon[66347]: Reconfiguring osd.7 (monmap changed)... 2024-05-09T04:14:32.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-09T04:14:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:32 smithi132 ceph-mon[66347]: Reconfiguring daemon osd.7 on smithi132 2024-05-09T04:14:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:32 smithi132 ceph-mon[66347]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:32.417 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:14:32.417 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (3m) 27s ago 25m 18.9M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:14:32.418 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (2m) 44s ago 24m 80.8M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:14:32.418 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (4m) 27s ago 24m 59.4M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:14:32.418 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (4m) 44s ago 29m 463M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:14:32.418 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (14m) 27s ago 31m 510M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:14:32.418 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (60s) 27s ago 31m 42.0M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:14:32.419 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (111s) 44s ago 29m 36.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:14:32.419 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (33s) 27s ago 30m 19.4M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:14:32.419 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (3m) 27s ago 25m 8586k - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:14:32.419 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (3m) 44s ago 25m 17.3M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:14:32.419 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (29m) 27s ago 29m 54.9M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:14:32.419 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (29m) 27s ago 29m 59.1M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:14:32.420 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (28m) 27s ago 28m 51.3M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:14:32.420 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (28m) 27s ago 28m 54.6M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:14:32.420 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (27m) 44s ago 27m 58.2M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:14:32.420 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (27m) 44s ago 27m 52.7M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:14:32.420 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (27m) 44s ago 27m 53.5M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:14:32.420 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (26m) 44s ago 26m 57.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:14:32.421 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (3m) 44s ago 26m 65.5M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:14:32.421 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (24m) 27s ago 24m 79.1M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:14:32.421 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (24m) 44s ago 24m 80.6M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:14:32.421 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[83176]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:32.422 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:32.422 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:32.422 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[83176]: Reconfiguring osd.7 (monmap changed)... 2024-05-09T04:14:32.422 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-09T04:14:32.422 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:32.423 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[83176]: Reconfiguring daemon osd.7 on smithi132 2024-05-09T04:14:32.423 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[83176]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:32.423 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[88406]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:32.423 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:32.423 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:32.424 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[88406]: Reconfiguring osd.7 (monmap changed)... 2024-05-09T04:14:32.424 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-09T04:14:32.424 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:32.424 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[88406]: Reconfiguring daemon osd.7 on smithi132 2024-05-09T04:14:32.424 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:32 smithi078 ceph-mon[88406]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:32.830 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:14:32.830 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:14:32.830 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:14:32.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:14:32.830 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 5 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:14:32.831 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:14:33.175 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:14:33.175 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:14:33.176 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:14:33.176 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi078", 2024-05-09T04:14:33.176 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-05-09T04:14:33.176 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-05-09T04:14:33.176 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-05-09T04:14:33.176 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/2 daemons upgraded", 2024-05-09T04:14:33.176 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mon daemons", 2024-05-09T04:14:33.176 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:14:33.176 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:14:34.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:33 smithi132 ceph-mon[66347]: from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:34.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:33 smithi132 ceph-mon[66347]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:34.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:33 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:33 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:33 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T04:14:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:33 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:34.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:33 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2654790602' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[83176]: from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[83176]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T04:14:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:34.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/2654790602' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[88406]: from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[88406]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T04:14:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:34.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:33 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/2654790602' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: Reconfiguring rgw.foo.smithi132.pcutwm (monmap changed)... 2024-05-09T04:14:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: Reconfiguring daemon rgw.foo.smithi132.pcutwm on smithi132 2024-05-09T04:14:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='client.44190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: Reconfiguring rgw.foo.smithi132.pcutwm (monmap changed)... 2024-05-09T04:14:35.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: Reconfiguring daemon rgw.foo.smithi132.pcutwm on smithi132 2024-05-09T04:14:35.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:35.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='client.44190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:35.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:14:35.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:14:35.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:14:35.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:14:35.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:14:35.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:14:35.017 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:14:35.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:14:35.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:14:35.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:14:35.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:14:35.019 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:14:35.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:14:35.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:14:35.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:14:35.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:14:35.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:14:35.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:14:35.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:14:35.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:14:35.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:14:35.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:14:35.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:14:35.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:14:35.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:14:35.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:14:35.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:14:35.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:14:35.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:14:35.023 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:14:35.023 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:35.023 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:14:35.023 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:14:35.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:35.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:14:35.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:14:35.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:35.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:14:35.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:14:35.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:35.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:14:35.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: Reconfiguring rgw.foo.smithi132.pcutwm (monmap changed)... 2024-05-09T04:14:35.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: Reconfiguring daemon rgw.foo.smithi132.pcutwm on smithi132 2024-05-09T04:14:35.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:35.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='client.44190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:14:35.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:14:35.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:14:35.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-09T04:14:35.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-09T04:14:35.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:14:35.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:14:35.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:14:35.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:14:35.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:14:35.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:14:35.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:14:35.260 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:36.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mon 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all crash 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mds 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all nfs 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all node-exporter 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all prometheus 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all alertmanager 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all grafana 2024-05-09T04:14:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all loki 2024-05-09T04:14:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all promtail 2024-05-09T04:14:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Finalizing container_image settings 2024-05-09T04:14:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[88406]: Upgrade: Complete! 2024-05-09T04:14:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mon 2024-05-09T04:14:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all crash 2024-05-09T04:14:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mds 2024-05-09T04:14:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:14:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-09T04:14:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:14:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all nfs 2024-05-09T04:14:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:14:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all node-exporter 2024-05-09T04:14:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all prometheus 2024-05-09T04:14:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all alertmanager 2024-05-09T04:14:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all grafana 2024-05-09T04:14:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all loki 2024-05-09T04:14:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all promtail 2024-05-09T04:14:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Finalizing container_image settings 2024-05-09T04:14:36.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:35 smithi078 ceph-mon[83176]: Upgrade: Complete! 2024-05-09T04:14:36.159 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:36.159 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mon 2024-05-09T04:14:36.159 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all crash 2024-05-09T04:14:36.159 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mds 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all nfs 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all node-exporter 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all prometheus 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all alertmanager 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all grafana 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all loki 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all promtail 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Finalizing container_image settings 2024-05-09T04:14:36.160 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:35 smithi132 ceph-mon[66347]: Upgrade: Complete! 2024-05-09T04:14:37.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:36 smithi132 ceph-mon[66347]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:37.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:36 smithi078 ceph-mon[88406]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:36 smithi078 ceph-mon[83176]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:39.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:38 smithi078 ceph-mon[83176]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:39.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:38 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:39.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:38 smithi078 ceph-mon[88406]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:39.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:38 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:39.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:38 smithi132 ceph-mon[66347]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:39.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:38 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:14:39.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:14:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:14:39] "GET /metrics HTTP/1.1" 200 38096 "" "Prometheus/2.43.0" 2024-05-09T04:14:40.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:39 smithi132 ceph-mon[66347]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:40.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:39 smithi078 ceph-mon[83176]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:40.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:39 smithi078 ceph-mon[88406]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:42.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:41 smithi132 ceph-mon[66347]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:42.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:41 smithi078 ceph-mon[83176]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:42.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:41 smithi078 ceph-mon[88406]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:43 smithi132 ceph-mon[66347]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:14:44.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:43 smithi078 ceph-mon[83176]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:44.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:14:44.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:43 smithi078 ceph-mon[88406]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:44.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:14:45.222 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:44 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:45.223 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:44 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:45.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:44 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:46.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:45 smithi132 ceph-mon[66347]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:46.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:45 smithi078 ceph-mon[83176]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:46.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:45 smithi078 ceph-mon[88406]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:48.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:47 smithi132 ceph-mon[66347]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:48.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:47 smithi078 ceph-mon[83176]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:47 smithi078 ceph-mon[88406]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:49.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:14:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:14:49] "GET /metrics HTTP/1.1" 200 38096 "" "Prometheus/2.43.0" 2024-05-09T04:14:50.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:49 smithi132 ceph-mon[66347]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:50.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:49 smithi078 ceph-mon[83176]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:50.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:49 smithi078 ceph-mon[88406]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:52.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:51 smithi132 ceph-mon[66347]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:52.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:51 smithi078 ceph-mon[83176]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:52.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:51 smithi078 ceph-mon[88406]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:54.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:53 smithi132 ceph-mon[66347]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:54.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:53 smithi078 ceph-mon[83176]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:54.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:53 smithi078 ceph-mon[88406]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:55.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:54 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:55.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:54 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:55.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:54 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:14:56.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:55 smithi132 ceph-mon[66347]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:56.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:55 smithi078 ceph-mon[83176]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:56.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:55 smithi078 ceph-mon[88406]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:14:58.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:57 smithi132 ceph-mon[66347]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:58.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:57 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:14:58.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:57 smithi078 ceph-mon[88406]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:58.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:57 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:14:58.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:57 smithi078 ceph-mon[83176]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:14:58.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:57 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:14:59.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:14:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:14:59] "GET /metrics HTTP/1.1" 200 38097 "" "Prometheus/2.43.0" 2024-05-09T04:15:00.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:14:59 smithi132 ceph-mon[66347]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:00.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:14:59 smithi078 ceph-mon[83176]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:00.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:14:59 smithi078 ceph-mon[88406]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:02.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:01 smithi132 ceph-mon[66347]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:02.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:01 smithi078 ceph-mon[88406]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:02.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:01 smithi078 ceph-mon[83176]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:04.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:03 smithi132 ceph-mon[66347]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:04.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:03 smithi132 ceph-mon[66347]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:04.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:03 smithi078 ceph-mon[83176]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:04.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:03 smithi078 ceph-mon[83176]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:04.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:03 smithi078 ceph-mon[88406]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:04.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:03 smithi078 ceph-mon[88406]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:04.299 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T04:15:05.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:05 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:05.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:05 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:05.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:05 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:06.249 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:15:06.249 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (3m) 61s ago 25m 18.9M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:15:06.249 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (3m) 78s ago 25m 80.8M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:15:06.249 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (4m) 61s ago 24m 59.4M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:15:06.249 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (4m) 78s ago 30m 463M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:15:06.249 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (14m) 61s ago 32m 510M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:15:06.249 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (93s) 61s ago 32m 42.0M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:15:06.249 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (2m) 78s ago 30m 36.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (67s) 61s ago 30m 19.4M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (4m) 61s ago 26m 8586k - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (4m) 78s ago 26m 17.3M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (30m) 61s ago 30m 54.9M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (29m) 61s ago 29m 59.1M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (29m) 61s ago 29m 51.3M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (28m) 61s ago 28m 54.6M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (28m) 78s ago 28m 58.2M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (28m) 78s ago 28m 52.7M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (27m) 78s ago 27m 53.5M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (27m) 78s ago 27m 57.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (3m) 78s ago 26m 65.5M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:15:06.250 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (24m) 61s ago 24m 79.1M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:15:06.251 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (24m) 78s ago 24m 80.6M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:15:06.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:06 smithi078 ceph-mon[83176]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:06.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:06 smithi078 ceph-mon[88406]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:06.402 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:06 smithi132 ceph-mon[66347]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:07.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-05-09T04:15:07.307 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:07 smithi132 ceph-mon[66347]: from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:07.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:07 smithi078 ceph-mon[88406]: from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:07.363 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:07 smithi078 ceph-mon[83176]: from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:08.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:08 smithi078 ceph-mon[88406]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:08.924 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:08 smithi078 ceph-mon[83176]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:09.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:08 smithi132 ceph-mon[66347]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:09.249 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:15:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:15:09] "GET /metrics HTTP/1.1" 200 38097 "" "Prometheus/2.43.0" 2024-05-09T04:15:09.249 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:15:09.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-05-09T04:15:09.809 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:09 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/1578899825' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:09.809 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:09 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/1578899825' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:10.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:09 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/1578899825' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:11.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:10 smithi132 ceph-mon[66347]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:11.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:10 smithi078 ceph-mon[83176]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:11.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:10 smithi078 ceph-mon[88406]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:11.828 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)" 2024-05-09T04:15:12.425 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-05-09T04:15:12.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:12 smithi078 ceph-mon[88406]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:12.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:12 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/126379127' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:12.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:12 smithi078 ceph-mon[83176]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:12.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:12 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/126379127' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:13.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:12 smithi132 ceph-mon[66347]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:13.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:12 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/126379127' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:13.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:15:13.936 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:15:14.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:15:15.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:14 smithi132 ceph-mon[66347]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:15.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:14 smithi078 ceph-mon[88406]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:15.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:14 smithi078 ceph-mon[83176]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:15.730 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:15 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:15.731 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:15 smithi078 ceph-mon[88406]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:15.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:15 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:15.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:15 smithi078 ceph-mon[83176]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:16.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:15 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:16.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:15 smithi132 ceph-mon[66347]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:16.600 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:15:16.967 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:16 smithi078 ceph-mon[83176]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:16.967 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:16 smithi078 ceph-mon[88406]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:16 smithi132 ceph-mon[66347]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:17.268 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade status' 2024-05-09T04:15:18.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:18 smithi078 ceph-mon[88406]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:18.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:18 smithi078 ceph-mon[83176]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:19.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:18 smithi132 ceph-mon[66347]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:19.226 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:15:19.227 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": null, 2024-05-09T04:15:19.227 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": false, 2024-05-09T04:15:19.227 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "", 2024-05-09T04:15:19.227 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:15:19.227 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": null, 2024-05-09T04:15:19.227 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-05-09T04:15:19.228 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:15:19.228 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:15:19.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:15:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:15:19] "GET /metrics HTTP/1.1" 200 38094 "" "Prometheus/2.43.0" 2024-05-09T04:15:19.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph health detail' 2024-05-09T04:15:21.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:20 smithi132 ceph-mon[66347]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:21.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:20 smithi132 ceph-mon[66347]: from='client.44226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:21.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:20 smithi078 ceph-mon[83176]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:21.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:20 smithi078 ceph-mon[83176]: from='client.44226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:20 smithi078 ceph-mon[88406]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:20 smithi078 ceph-mon[88406]: from='client.44226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:21.879 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-05-09T04:15:22.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-05-09T04:15:22.889 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:22 smithi078 ceph-mon[83176]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:22.889 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:22 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/886552427' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:15:22.890 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:22 smithi078 ceph-mon[88406]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:22.890 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:22 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/886552427' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:15:23.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:22 smithi132 ceph-mon[66347]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:23.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:22 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/886552427' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:15:25.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:24 smithi132 ceph-mon[66347]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:25.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:24 smithi078 ceph-mon[88406]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:25.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:24 smithi078 ceph-mon[83176]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:26.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:25 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:26.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:25 smithi132 ceph-mon[66347]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:26.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:25 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:26.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:25 smithi078 ceph-mon[83176]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:26.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:25 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:26.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:25 smithi078 ceph-mon[88406]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:26.484 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:15:27.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:26 smithi132 ceph-mon[66347]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:27.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:27.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:15:27.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:26 smithi078 ceph-mon[83176]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:27.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:27.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:15:27.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:26 smithi078 ceph-mon[88406]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:27.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:27.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:15:27.122 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-09T04:15:28.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:27 smithi132 ceph-mon[66347]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:15:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:27 smithi078 ceph-mon[83176]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:15:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:28.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:27 smithi078 ceph-mon[88406]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:15:28.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:28 smithi132 ceph-mon[66347]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:15:29.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:28 smithi078 ceph-mon[88406]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:29.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:15:29.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:28 smithi078 ceph-mon[83176]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:29.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:15:29.082 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:15:29.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:15:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:15:29] "GET /metrics HTTP/1.1" 200 38096 "" "Prometheus/2.43.0" 2024-05-09T04:15:29.786 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:15:29.786 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (4m) 84s ago 26m 18.9M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:15:29.786 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (3m) 102s ago 25m 80.8M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:15:29.786 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (5m) 84s ago 25m 59.4M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (5m) 102s ago 30m 463M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (15m) 84s ago 32m 510M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (117s) 84s ago 32m 42.0M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (2m) 102s ago 30m 36.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (91s) 84s ago 31m 19.4M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (4m) 84s ago 26m 8586k - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (4m) 102s ago 26m 17.3M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (30m) 84s ago 30m 54.9M 2992M 17.2.0 e1d6a67b021e b2bcec2a31a0 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (30m) 84s ago 29m 59.1M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (29m) 84s ago 29m 51.3M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (29m) 84s ago 29m 54.6M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (28m) 102s ago 28m 58.2M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (28m) 102s ago 28m 52.7M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (28m) 102s ago 28m 53.5M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:15:29.787 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (27m) 102s ago 27m 57.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:15:29.788 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (4m) 102s ago 26m 65.5M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:15:29.788 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (25m) 84s ago 25m 79.1M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:15:29.788 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (25m) 102s ago 25m 80.6M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:15:30.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:29 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:15:30.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:29 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:15:30.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:29 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:30.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:29 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:15:30.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:29 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:15:30.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:29 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:30.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:29 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:15:30.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:29 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:15:30.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:29 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:30.215 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:15:30.215 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:15:30.215 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:15:30.215 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:15:30.215 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:15:30.215 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:15:30.215 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:15:30.215 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:15:30.216 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-09T04:15:30.216 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:15:30.216 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:15:30.216 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:15:30.216 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:15:30.216 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:15:30.216 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-09T04:15:30.216 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 5 2024-05-09T04:15:30.216 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:15:30.216 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:15:30.567 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:15:30.567 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", 2024-05-09T04:15:30.567 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:15:30.567 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-05-09T04:15:30.567 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:15:30.567 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "", 2024-05-09T04:15:30.567 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 image", 2024-05-09T04:15:30.567 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:15:30.567 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:15:30.957 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:30 smithi078 ceph-mon[88406]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:15:30.957 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:30 smithi078 ceph-mon[88406]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:30.957 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:30 smithi078 ceph-mon[88406]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:30.957 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:30 smithi078 ceph-mon[88406]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:30.957 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:30 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/3299390716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:30.958 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:30 smithi078 ceph-mon[83176]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:15:30.958 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:30 smithi078 ceph-mon[83176]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:30.958 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:30 smithi078 ceph-mon[83176]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:30.958 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:30 smithi078 ceph-mon[83176]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:30.958 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:30 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/3299390716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:31.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:30 smithi132 ceph-mon[66347]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:15:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:30 smithi132 ceph-mon[66347]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:30 smithi132 ceph-mon[66347]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:30 smithi132 ceph-mon[66347]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:31.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:30 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/3299390716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:31.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[83176]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:31.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[83176]: from='client.54205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:31.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:31.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:15:31.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:31.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:31.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:31.962 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:31.962 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:31.962 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:31.962 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-09T04:15:31.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[88406]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:31.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[88406]: from='client.54205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:31.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:31.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:15:31.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:31.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:31.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:31.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:31.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:31.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:31.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:31 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-09T04:15:32.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:31 smithi132 ceph-mon[66347]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:31 smithi132 ceph-mon[66347]: from='client.54205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:15:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:31 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:31 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:15:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:31 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:31 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:31 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:31 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:31 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:31 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:32.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:31 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-09T04:15:33.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[83176]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:33.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[83176]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:15:33.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[83176]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:15:33.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mgr 2024-05-09T04:15:33.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mon 2024-05-09T04:15:33.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all crash 2024-05-09T04:15:33.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-09T04:15:33.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[83176]: Upgrade: osd.0 is safe to restart 2024-05-09T04:15:33.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:33.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-09T04:15:33.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:15:33.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[88406]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:33.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[88406]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:15:33.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[88406]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:15:33.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mgr 2024-05-09T04:15:33.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mon 2024-05-09T04:15:33.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all crash 2024-05-09T04:15:33.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-09T04:15:33.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[88406]: Upgrade: osd.0 is safe to restart 2024-05-09T04:15:33.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:33.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-09T04:15:33.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:15:33.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:32 smithi132 ceph-mon[66347]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:33.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:32 smithi132 ceph-mon[66347]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:15:33.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:32 smithi132 ceph-mon[66347]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:15:33.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:32 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mgr 2024-05-09T04:15:33.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:32 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mon 2024-05-09T04:15:33.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:32 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all crash 2024-05-09T04:15:33.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:32 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-09T04:15:33.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:32 smithi132 ceph-mon[66347]: Upgrade: osd.0 is safe to restart 2024-05-09T04:15:33.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:33.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-09T04:15:33.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:15:33.988 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:33 smithi078 ceph-mon[83176]: Upgrade: Updating osd.0 2024-05-09T04:15:33.988 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:33 smithi078 ceph-mon[83176]: Deploying daemon osd.0 on smithi078 2024-05-09T04:15:33.989 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:33 smithi078 ceph-mon[88406]: Upgrade: Updating osd.0 2024-05-09T04:15:33.989 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:33 smithi078 ceph-mon[88406]: Deploying daemon osd.0 on smithi078 2024-05-09T04:15:34.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:33 smithi132 ceph-mon[66347]: Upgrade: Updating osd.0 2024-05-09T04:15:34.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:33 smithi132 ceph-mon[66347]: Deploying daemon osd.0 on smithi078 2024-05-09T04:15:34.861 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:34 smithi078 ceph-mon[88406]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:34.861 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:34 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:34.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:34 smithi078 ceph-mon[83176]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:34.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:34 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:35.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:34 smithi132 ceph-mon[66347]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:15:35.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:34 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:36.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:35 smithi078 ceph-mon[88406]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:36.011 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:35 smithi078 systemd[1]: Stopping Ceph osd.0 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:15:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:35 smithi078 ceph-mon[83176]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:36.155 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:35 smithi132 ceph-mon[66347]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:36.511 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0[30489]: 2024-05-09T04:15:36.046+0000 7f0daa0a9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:15:36.511 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0[30489]: 2024-05-09T04:15:36.046+0000 7f0daa0a9700 -1 osd.0 113 *** Got signal Terminated *** 2024-05-09T04:15:36.511 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:36 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0[30489]: 2024-05-09T04:15:36.046+0000 7f0daa0a9700 -1 osd.0 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:15:37.169 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:36 smithi078 ceph-mon[83176]: osd.0 marked itself down and dead 2024-05-09T04:15:37.170 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:36 smithi078 ceph-mon[88406]: osd.0 marked itself down and dead 2024-05-09T04:15:37.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:36 smithi132 ceph-mon[66347]: osd.0 marked itself down and dead 2024-05-09T04:15:37.439 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:37 smithi078 podman[95448]: 2024-05-09 04:15:37.169981845 +0000 UTC m=+1.227729133 container died b2bcec2a31a0d005e3fb59a67c5a33027024990739d7faf68b8daf333907170e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=HEAD, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, version=8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD) 2024-05-09T04:15:37.761 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:37 smithi078 podman[95448]: 2024-05-09 04:15:37.538397083 +0000 UTC m=+1.596144360 container cleanup b2bcec2a31a0d005e3fb59a67c5a33027024990739d7faf68b8daf333907170e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, distribution-scope=public, ceph=True, io.openshift.expose-services=, io.buildah.version=1.19.8, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vendor=Red Hat, Inc., RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, release=754) 2024-05-09T04:15:37.761 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:37 smithi078 bash[95448]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0 2024-05-09T04:15:38.144 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:37 smithi078 ceph-mon[83176]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:15:38.145 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:37 smithi078 ceph-mon[83176]: osdmap e114: 8 total, 7 up, 8 in 2024-05-09T04:15:38.145 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:37 smithi078 ceph-mon[83176]: pgmap v61: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:15:38.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:37 smithi078 ceph-mon[88406]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:15:38.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:37 smithi078 ceph-mon[88406]: osdmap e114: 8 total, 7 up, 8 in 2024-05-09T04:15:38.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:37 smithi078 ceph-mon[88406]: pgmap v61: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:15:38.145 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:37 smithi078 podman[95465]: 2024-05-09 04:15:37.831278746 +0000 UTC m=+0.659644283 container remove b2bcec2a31a0d005e3fb59a67c5a33027024990739d7faf68b8daf333907170e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.openshift.expose-services=, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, name=centos-stream, release=754, version=8, io.buildah.version=1.19.8, ceph=True, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, distribution-scope=public, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-05-09T04:15:38.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:37 smithi132 ceph-mon[66347]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:15:38.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:37 smithi132 ceph-mon[66347]: osdmap e114: 8 total, 7 up, 8 in 2024-05-09T04:15:38.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:37 smithi132 ceph-mon[66347]: pgmap v61: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:15:38.401 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:38 smithi078 podman[95558]: 2024-05-09 04:15:38.047923296 +0000 UTC m=+0.018303196 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:15:38.401 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:38 smithi078 podman[95558]: 2024-05-09 04:15:38.173879445 +0000 UTC m=+0.144259391 container create 41980686d464de0bd12d183cb91a960f63a3d5a5cebbe7b99567864df86c707f (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-05-09T04:15:38.699 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:38 smithi078 podman[95558]: 2024-05-09 04:15:38.401964789 +0000 UTC m=+0.372344690 container init 41980686d464de0bd12d183cb91a960f63a3d5a5cebbe7b99567864df86c707f (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504) 2024-05-09T04:15:38.700 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:38 smithi078 podman[95558]: 2024-05-09 04:15:38.405849712 +0000 UTC m=+0.376229619 container start 41980686d464de0bd12d183cb91a960f63a3d5a5cebbe7b99567864df86c707f (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-05-09T04:15:38.700 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:38 smithi078 podman[95558]: 2024-05-09 04:15:38.509586285 +0000 UTC m=+0.479966227 container attach 41980686d464de0bd12d183cb91a960f63a3d5a5cebbe7b99567864df86c707f (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240507, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:15:39.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:38 smithi078 ceph-mon[83176]: osdmap e115: 8 total, 7 up, 8 in 2024-05-09T04:15:39.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:38 smithi078 ceph-mon[88406]: osdmap e115: 8 total, 7 up, 8 in 2024-05-09T04:15:39.011 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:38 smithi078 podman[95584]: 2024-05-09 04:15:38.730280755 +0000 UTC m=+0.019174332 container died 41980686d464de0bd12d183cb91a960f63a3d5a5cebbe7b99567864df86c707f (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0) 2024-05-09T04:15:39.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:38 smithi132 ceph-mon[66347]: osdmap e115: 8 total, 7 up, 8 in 2024-05-09T04:15:39.337 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:15:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:15:39] "GET /metrics HTTP/1.1" 200 38096 "" "Prometheus/2.43.0" 2024-05-09T04:15:39.603 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:39 smithi078 podman[95584]: 2024-05-09 04:15:39.338472356 +0000 UTC m=+0.627365920 container remove 41980686d464de0bd12d183cb91a960f63a3d5a5cebbe7b99567864df86c707f (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, GIT_CLEAN=True) 2024-05-09T04:15:39.604 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:39 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.0.service: Deactivated successfully. 2024-05-09T04:15:39.604 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:39 smithi078 systemd[1]: Stopped Ceph osd.0 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:15:39.604 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:39 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.0.service: Consumed 14.126s CPU time. 2024-05-09T04:15:39.604 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:39 smithi078 systemd[1]: Starting Ceph osd.0 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:15:39.911 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:39 smithi078 ceph-mon[83176]: pgmap v63: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-09T04:15:39.912 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:39 smithi078 ceph-mon[88406]: pgmap v63: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-09T04:15:39.912 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:39 smithi078 podman[95691]: 2024-05-09 04:15:39.705930331 +0000 UTC m=+0.018955342 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:15:39.912 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:39 smithi078 podman[95691]: 2024-05-09 04:15:39.814556198 +0000 UTC m=+0.127581192 container create 1b0498e9ac6dbcfad681c7886617344a52811d88aa4e6a23f09a7af67783ecc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507) 2024-05-09T04:15:40.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:39 smithi132 ceph-mon[66347]: pgmap v63: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-09T04:15:40.261 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 podman[95691]: 2024-05-09 04:15:40.050782316 +0000 UTC m=+0.363807310 container init 1b0498e9ac6dbcfad681c7886617344a52811d88aa4e6a23f09a7af67783ecc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate, GIT_CLEAN=True, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:15:40.261 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 podman[95691]: 2024-05-09 04:15:40.055124359 +0000 UTC m=+0.368149363 container start 1b0498e9ac6dbcfad681c7886617344a52811d88aa4e6a23f09a7af67783ecc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:15:40.261 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 podman[95691]: 2024-05-09 04:15:40.134009787 +0000 UTC m=+0.447034806 container attach 1b0498e9ac6dbcfad681c7886617344a52811d88aa4e6a23f09a7af67783ecc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-05-09T04:15:40.796 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:40 smithi078 ceph-mon[83176]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 21 pgs degraded (PG_DEGRADED) 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate[95714]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate[95714]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 bash[95691]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 bash[95691]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 bash[95691]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 bash[95691]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 bash[95691]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 bash[95691]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 bash[95691]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 bash[95691]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 bash[95691]: --> ceph-volume raw activate successful for osd ID: 0 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate[95714]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate[95714]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-09T04:15:40.797 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate[95714]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-09T04:15:40.798 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate[95714]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-09T04:15:40.798 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate[95714]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-05-09T04:15:40.798 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate[95714]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-09T04:15:40.798 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate[95714]: --> ceph-volume raw activate successful for osd ID: 0 2024-05-09T04:15:40.798 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:40 smithi078 podman[95691]: 2024-05-09 04:15:40.573593823 +0000 UTC m=+0.886618818 container died 1b0498e9ac6dbcfad681c7886617344a52811d88aa4e6a23f09a7af67783ecc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.license=GPLv2, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux ) 2024-05-09T04:15:41.174 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:40 smithi078 ceph-mon[88406]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 21 pgs degraded (PG_DEGRADED) 2024-05-09T04:15:41.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:40 smithi132 ceph-mon[66347]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 21 pgs degraded (PG_DEGRADED) 2024-05-09T04:15:41.422 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:41 smithi078 podman[95853]: 2024-05-09 04:15:41.171431606 +0000 UTC m=+0.589978508 container remove 1b0498e9ac6dbcfad681c7886617344a52811d88aa4e6a23f09a7af67783ecc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:15:41.422 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:41 smithi078 podman[95915]: 2024-05-09 04:15:41.288696026 +0000 UTC m=+0.018257535 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:15:41.422 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:41 smithi078 podman[95915]: 2024-05-09 04:15:41.422314019 +0000 UTC m=+0.151875565 container create ba788dd1e3fe2cb71644c00b04e269085d8b53421890c2d91b7647b9cf99e8e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS) 2024-05-09T04:15:41.682 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:41 smithi078 podman[95915]: 2024-05-09 04:15:41.658292738 +0000 UTC m=+0.387854250 container init ba788dd1e3fe2cb71644c00b04e269085d8b53421890c2d91b7647b9cf99e8e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-05-09T04:15:41.683 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:41 smithi078 podman[95915]: 2024-05-09 04:15:41.661833997 +0000 UTC m=+0.391395506 container start ba788dd1e3fe2cb71644c00b04e269085d8b53421890c2d91b7647b9cf99e8e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, ceph=True) 2024-05-09T04:15:42.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:41 smithi078 ceph-mon[83176]: pgmap v64: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-09T04:15:42.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:41 smithi078 ceph-mon[88406]: pgmap v64: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-09T04:15:42.012 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:41 smithi078 ceph-osd[95934]: -- 172.21.15.78:0/2638838447 <== mon.1 v2:172.21.15.78:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55620c403680 con 0x55620c3cd000 2024-05-09T04:15:42.012 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:41 smithi078 bash[95915]: ba788dd1e3fe2cb71644c00b04e269085d8b53421890c2d91b7647b9cf99e8e0 2024-05-09T04:15:42.012 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:41 smithi078 systemd[1]: Started Ceph osd.0 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:15:42.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:41 smithi132 ceph-mon[66347]: pgmap v64: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-09T04:15:42.511 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:42 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0[95930]: 2024-05-09T04:15:42.205+0000 7f6212a347c0 -1 Falling back to public interface 2024-05-09T04:15:42.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:42.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:42.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:42.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:15:42.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:42 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:42.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:42 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:42.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:42 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:42.904 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:42 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:15:43.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:43.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:43.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:43.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:15:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:43 smithi132 ceph-mon[66347]: pgmap v65: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-09T04:15:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:15:44.258 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:43 smithi078 ceph-mon[83176]: pgmap v65: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-09T04:15:44.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:44.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:15:44.259 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:43 smithi078 ceph-mon[88406]: pgmap v65: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-05-09T04:15:44.259 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:44.259 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:15:45.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:44 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:45.254 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:44 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:45.255 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:44 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:45.255 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:44 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0[95930]: 2024-05-09T04:15:44.952+0000 7f6212a347c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-05-09T04:15:45.255 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:44 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0[95930]: 2024-05-09T04:15:44.980+0000 7f6212a347c0 -1 osd.0 113 log_to_monitors true 2024-05-09T04:15:46.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:45 smithi078 ceph-mon[83176]: pgmap v66: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 503 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-09T04:15:46.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:45 smithi078 ceph-mon[83176]: from='osd.0 [v2:172.21.15.78:6802/1453500380,v1:172.21.15.78:6803/1453500380]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-09T04:15:46.004 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:45 smithi078 ceph-mon[88406]: pgmap v66: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 503 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-09T04:15:46.004 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:45 smithi078 ceph-mon[88406]: from='osd.0 [v2:172.21.15.78:6802/1453500380,v1:172.21.15.78:6803/1453500380]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-09T04:15:46.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:45 smithi132 ceph-mon[66347]: pgmap v66: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 503 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-09T04:15:46.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:45 smithi132 ceph-mon[66347]: from='osd.0 [v2:172.21.15.78:6802/1453500380,v1:172.21.15.78:6803/1453500380]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-09T04:15:46.261 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:15:46 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0[95930]: 2024-05-09T04:15:46.004+0000 7f6203e8b700 -1 osd.0 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T04:15:47.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:46 smithi132 ceph-mon[66347]: from='osd.0 [v2:172.21.15.78:6802/1453500380,v1:172.21.15.78:6803/1453500380]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-09T04:15:47.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:46 smithi132 ceph-mon[66347]: osdmap e116: 8 total, 7 up, 8 in 2024-05-09T04:15:47.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:46 smithi132 ceph-mon[66347]: from='osd.0 [v2:172.21.15.78:6802/1453500380,v1:172.21.15.78:6803/1453500380]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:15:47.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:46 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:47.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:46 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:47.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:46 smithi078 ceph-mon[88406]: from='osd.0 [v2:172.21.15.78:6802/1453500380,v1:172.21.15.78:6803/1453500380]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-09T04:15:47.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:46 smithi078 ceph-mon[88406]: osdmap e116: 8 total, 7 up, 8 in 2024-05-09T04:15:47.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:46 smithi078 ceph-mon[88406]: from='osd.0 [v2:172.21.15.78:6802/1453500380,v1:172.21.15.78:6803/1453500380]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:15:47.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:46 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:47.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:46 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:47.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:46 smithi078 ceph-mon[83176]: from='osd.0 [v2:172.21.15.78:6802/1453500380,v1:172.21.15.78:6803/1453500380]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-09T04:15:47.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:46 smithi078 ceph-mon[83176]: osdmap e116: 8 total, 7 up, 8 in 2024-05-09T04:15:47.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:46 smithi078 ceph-mon[83176]: from='osd.0 [v2:172.21.15.78:6802/1453500380,v1:172.21.15.78:6803/1453500380]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:15:47.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:46 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:47.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:46 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:48.192 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[88406]: pgmap v68: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 448 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-09T04:15:48.192 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[88406]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:15:48.192 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[88406]: osd.0 [v2:172.21.15.78:6802/1453500380,v1:172.21.15.78:6803/1453500380] boot 2024-05-09T04:15:48.193 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[88406]: osdmap e117: 8 total, 8 up, 8 in 2024-05-09T04:15:48.193 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:15:48.193 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:48.193 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:48.193 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[88406]: osdmap e118: 8 total, 8 up, 8 in 2024-05-09T04:15:48.193 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[83176]: pgmap v68: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 448 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-09T04:15:48.193 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[83176]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:15:48.193 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[83176]: osd.0 [v2:172.21.15.78:6802/1453500380,v1:172.21.15.78:6803/1453500380] boot 2024-05-09T04:15:48.193 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[83176]: osdmap e117: 8 total, 8 up, 8 in 2024-05-09T04:15:48.193 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:15:48.193 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:48.193 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:48.194 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:47 smithi078 ceph-mon[83176]: osdmap e118: 8 total, 8 up, 8 in 2024-05-09T04:15:48.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:47 smithi132 ceph-mon[66347]: pgmap v68: 161 pgs: 21 active+undersized+degraded, 37 active+undersized, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 448 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-05-09T04:15:48.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:47 smithi132 ceph-mon[66347]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:15:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:47 smithi132 ceph-mon[66347]: osd.0 [v2:172.21.15.78:6802/1453500380,v1:172.21.15.78:6803/1453500380] boot 2024-05-09T04:15:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:47 smithi132 ceph-mon[66347]: osdmap e117: 8 total, 8 up, 8 in 2024-05-09T04:15:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-09T04:15:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:47 smithi132 ceph-mon[66347]: osdmap e118: 8 total, 8 up, 8 in 2024-05-09T04:15:49.220 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:49 smithi078 ceph-mon[88406]: Health check update: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded (PG_DEGRADED) 2024-05-09T04:15:49.221 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:49 smithi078 ceph-mon[83176]: Health check update: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded (PG_DEGRADED) 2024-05-09T04:15:49.221 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:15:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:15:49] "GET /metrics HTTP/1.1" 200 38227 "" "Prometheus/2.43.0" 2024-05-09T04:15:49.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:49 smithi132 ceph-mon[66347]: Health check update: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded (PG_DEGRADED) 2024-05-09T04:15:50.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:50 smithi132 ceph-mon[66347]: pgmap v71: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 32/723 objects degraded (4.426%) 2024-05-09T04:15:50.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:50 smithi078 ceph-mon[88406]: pgmap v71: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 32/723 objects degraded (4.426%) 2024-05-09T04:15:50.380 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:50 smithi078 ceph-mon[83176]: pgmap v71: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 32/723 objects degraded (4.426%) 2024-05-09T04:15:51.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:15:51 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:15:51.048+0000 7f81013d6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-05-09T04:15:52.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:15:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:15:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: pgmap v72: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 32/723 objects degraded (4.426%) 2024-05-09T04:15:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:15:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-09T04:15:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-09T04:15:52.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:51 smithi132 ceph-mon[66347]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-05-09T04:15:52.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:52.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:52.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:15:52.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:15:52.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:52.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: pgmap v72: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 32/723 objects degraded (4.426%) 2024-05-09T04:15:52.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:15:52.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:52.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:52.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:52.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-09T04:15:52.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-09T04:15:52.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[83176]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-05-09T04:15:52.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:52.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:52.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:15:52.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:15:52.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:52.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: pgmap v72: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 32/723 objects degraded (4.426%) 2024-05-09T04:15:52.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:15:52.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:52.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:52.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:15:52.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-09T04:15:52.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-09T04:15:52.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:51 smithi078 ceph-mon[88406]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-05-09T04:15:53.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:53 smithi132 ceph-mon[66347]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded) 2024-05-09T04:15:53.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:53 smithi132 ceph-mon[66347]: Cluster is now healthy 2024-05-09T04:15:53.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:53 smithi078 ceph-mon[83176]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded) 2024-05-09T04:15:53.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:53 smithi078 ceph-mon[83176]: Cluster is now healthy 2024-05-09T04:15:53.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:53 smithi078 ceph-mon[88406]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded) 2024-05-09T04:15:53.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:53 smithi078 ceph-mon[88406]: Cluster is now healthy 2024-05-09T04:15:54.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:54 smithi078 ceph-mon[83176]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 736 B/s rd, 0 op/s 2024-05-09T04:15:54.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:54 smithi078 ceph-mon[88406]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 736 B/s rd, 0 op/s 2024-05-09T04:15:54.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:54 smithi132 ceph-mon[66347]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 736 B/s rd, 0 op/s 2024-05-09T04:15:55.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:55 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:55.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:55 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:55.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:55 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:15:56.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:56 smithi132 ceph-mon[66347]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:56.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:56 smithi078 ceph-mon[83176]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:56.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:56 smithi078 ceph-mon[88406]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:15:58.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:58 smithi132 ceph-mon[66347]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-09T04:15:58.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:58.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:15:58.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:15:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:58.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:58 smithi078 ceph-mon[88406]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-09T04:15:58.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:58.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:15:58.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:15:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:58.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:58 smithi078 ceph-mon[83176]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-09T04:15:58.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:15:58.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:15:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:15:59.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:15:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:15:59] "GET /metrics HTTP/1.1" 200 38247 "" "Prometheus/2.43.0" 2024-05-09T04:16:00.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:00 smithi132 ceph-mon[66347]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-09T04:16:00.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:00 smithi078 ceph-mon[83176]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-09T04:16:00.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:00 smithi078 ceph-mon[88406]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-09T04:16:00.935 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:16:01.609 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:16:01.609 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (4m) 15s ago 26m 18.4M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:16:01.609 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (4m) 2m ago 26m 80.8M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:16:01.609 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (5m) 15s ago 25m 64.7M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:16:01.609 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (5m) 2m ago 31m 463M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:16:01.609 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (15m) 15s ago 32m 519M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:16:01.609 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (2m) 15s ago 33m 45.8M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:16:01.609 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (3m) 2m ago 31m 36.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:16:01.609 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (2m) 15s ago 31m 36.3M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (5m) 15s ago 27m 8933k - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (5m) 2m ago 27m 17.3M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (20s) 15s ago 30m 15.0M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (30m) 15s ago 30m 59.8M 2992M 17.2.0 e1d6a67b021e 0ac5a1cb03d2 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (30m) 15s ago 30m 51.9M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (29m) 15s ago 29m 55.3M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (29m) 2m ago 29m 58.2M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (28m) 2m ago 28m 52.7M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (28m) 2m ago 28m 53.5M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (28m) 2m ago 28m 57.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (4m) 2m ago 27m 65.5M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (25m) 15s ago 25m 79.5M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:16:01.610 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (25m) 2m ago 25m 80.6M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 1 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:16:02.021 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:16:02.022 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:16:02.022 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:16:02.022 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-05-09T04:16:02.022 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 6 2024-05-09T04:16:02.022 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:16:02.022 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:16:02.362 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:16:02.363 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:16:02.363 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:16:02.363 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-05-09T04:16:02.363 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:16:02.363 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/8 daemons upgraded", 2024-05-09T04:16:02.363 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-05-09T04:16:02.363 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:16:02.363 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:16:02.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:02 smithi078 ceph-mon[88406]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:16:02.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:02 smithi078 ceph-mon[88406]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:02.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:02 smithi078 ceph-mon[88406]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:02.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:02 smithi078 ceph-mon[88406]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:02.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:02 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/1928807396' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:02.364 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:02 smithi078 ceph-mon[83176]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:16:02.364 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:02 smithi078 ceph-mon[83176]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:02.364 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:02 smithi078 ceph-mon[83176]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:02.364 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:02 smithi078 ceph-mon[83176]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:02.364 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:02 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/1928807396' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:02.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:02 smithi132 ceph-mon[66347]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:16:02.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:02 smithi132 ceph-mon[66347]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:02.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:02 smithi132 ceph-mon[66347]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:02.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:02 smithi132 ceph-mon[66347]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:02.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:02 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/1928807396' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:04.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:03 smithi132 ceph-mon[66347]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:04.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:03 smithi078 ceph-mon[83176]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:04.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:03 smithi078 ceph-mon[88406]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:05.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:04 smithi132 ceph-mon[66347]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:16:05.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:04 smithi078 ceph-mon[83176]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:16:05.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:04 smithi078 ceph-mon[88406]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:16:06.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:05 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:06.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:05 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:06.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:05 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:06.969 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:06 smithi078 ceph-mon[83176]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:06.969 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:06 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-09T04:16:06.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:06 smithi078 ceph-mon[88406]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:06.970 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:06 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-09T04:16:07.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:06 smithi132 ceph-mon[66347]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:07.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:06 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-09T04:16:08.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:07 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-09T04:16:08.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:07 smithi132 ceph-mon[66347]: Upgrade: osd.1 is safe to restart 2024-05-09T04:16:08.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:07 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:08.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:07 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-09T04:16:08.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:07 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:08.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:07 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-09T04:16:08.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:07 smithi078 ceph-mon[83176]: Upgrade: osd.1 is safe to restart 2024-05-09T04:16:08.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:07 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:08.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:07 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-09T04:16:08.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:07 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:08.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:07 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-09T04:16:08.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:07 smithi078 ceph-mon[88406]: Upgrade: osd.1 is safe to restart 2024-05-09T04:16:08.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:07 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:08.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:07 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-09T04:16:08.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:07 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:09.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:08 smithi132 ceph-mon[66347]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:16:09.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:08 smithi132 ceph-mon[66347]: Upgrade: Updating osd.1 2024-05-09T04:16:09.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:08 smithi132 ceph-mon[66347]: Deploying daemon osd.1 on smithi078 2024-05-09T04:16:09.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:08 smithi078 ceph-mon[83176]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:16:09.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:08 smithi078 ceph-mon[83176]: Upgrade: Updating osd.1 2024-05-09T04:16:09.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:08 smithi078 ceph-mon[83176]: Deploying daemon osd.1 on smithi078 2024-05-09T04:16:09.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:08 smithi078 ceph-mon[88406]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-09T04:16:09.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:08 smithi078 ceph-mon[88406]: Upgrade: Updating osd.1 2024-05-09T04:16:09.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:08 smithi078 ceph-mon[88406]: Deploying daemon osd.1 on smithi078 2024-05-09T04:16:09.471 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:16:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:16:09] "GET /metrics HTTP/1.1" 200 38247 "" "Prometheus/2.43.0" 2024-05-09T04:16:10.914 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:10 smithi078 ceph-mon[83176]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:10.915 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:10 smithi078 ceph-mon[88406]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:11.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:10 smithi132 ceph-mon[66347]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:11.231 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:10 smithi078 systemd[1]: Stopping Ceph osd.1 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:16:11.511 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:11 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1[33457]: 2024-05-09T04:16:11.230+0000 7f3852504700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:16:11.511 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:11 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1[33457]: 2024-05-09T04:16:11.230+0000 7f3852504700 -1 osd.1 118 *** Got signal Terminated *** 2024-05-09T04:16:11.511 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:11 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1[33457]: 2024-05-09T04:16:11.230+0000 7f3852504700 -1 osd.1 118 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:16:12.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:11 smithi132 ceph-mon[66347]: osd.1 marked itself down and dead 2024-05-09T04:16:12.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:11 smithi078 ceph-mon[88406]: osd.1 marked itself down and dead 2024-05-09T04:16:12.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:11 smithi078 ceph-mon[83176]: osd.1 marked itself down and dead 2024-05-09T04:16:12.305 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:12 smithi078 podman[100637]: 2024-05-09 04:16:12.054039948 +0000 UTC m=+0.972137830 container died 0ac5a1cb03d26abf27ede35c10fc4ad8c41d1b09d28b399ee30cbb5cb9d23534 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, vcs-type=git, ceph=True, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, maintainer=Guillaume Abrioux , RELEASE=HEAD, release=754, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-09T04:16:12.681 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:12 smithi078 podman[100637]: 2024-05-09 04:16:12.404982965 +0000 UTC m=+1.323080847 container cleanup 0ac5a1cb03d26abf27ede35c10fc4ad8c41d1b09d28b399ee30cbb5cb9d23534 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True) 2024-05-09T04:16:12.681 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:12 smithi078 bash[100637]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1 2024-05-09T04:16:13.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:12 smithi132 ceph-mon[66347]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:13.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:12 smithi132 ceph-mon[66347]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:16:13.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:12 smithi132 ceph-mon[66347]: osdmap e119: 8 total, 7 up, 8 in 2024-05-09T04:16:13.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:12 smithi078 ceph-mon[88406]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:13.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:12 smithi078 ceph-mon[88406]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:16:13.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:12 smithi078 ceph-mon[88406]: osdmap e119: 8 total, 7 up, 8 in 2024-05-09T04:16:13.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:12 smithi078 ceph-mon[83176]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:13.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:12 smithi078 ceph-mon[83176]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:16:13.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:12 smithi078 ceph-mon[83176]: osdmap e119: 8 total, 7 up, 8 in 2024-05-09T04:16:13.012 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:12 smithi078 podman[100666]: 2024-05-09 04:16:12.681196091 +0000 UTC m=+0.627558878 container remove 0ac5a1cb03d26abf27ede35c10fc4ad8c41d1b09d28b399ee30cbb5cb9d23534 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, ceph=True, io.buildah.version=1.19.8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , version=8, release=754, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, RELEASE=HEAD) 2024-05-09T04:16:13.284 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:13 smithi078 podman[100765]: 2024-05-09 04:16:12.934721458 +0000 UTC m=+0.018384094 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:16:13.284 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:13 smithi078 podman[100765]: 2024-05-09 04:16:13.057198836 +0000 UTC m=+0.140861456 container create b8496c5eb403604fe12315f5fc710a505f1e82cc71b67398e10af29c21e93221 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-05-09T04:16:13.284 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:13 smithi078 podman[100765]: 2024-05-09 04:16:13.284975847 +0000 UTC m=+0.368638472 container init b8496c5eb403604fe12315f5fc710a505f1e82cc71b67398e10af29c21e93221 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240507, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-05-09T04:16:13.583 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:13 smithi078 podman[100765]: 2024-05-09 04:16:13.288330238 +0000 UTC m=+0.371992866 container start b8496c5eb403604fe12315f5fc710a505f1e82cc71b67398e10af29c21e93221 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:16:13.583 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:13 smithi078 podman[100765]: 2024-05-09 04:16:13.351481213 +0000 UTC m=+0.435143850 container attach b8496c5eb403604fe12315f5fc710a505f1e82cc71b67398e10af29c21e93221 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-05-09T04:16:13.929 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:13 smithi078 ceph-mon[88406]: osdmap e120: 8 total, 7 up, 8 in 2024-05-09T04:16:13.930 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:13.930 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:16:13.930 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:13 smithi078 ceph-mon[83176]: osdmap e120: 8 total, 7 up, 8 in 2024-05-09T04:16:13.930 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:13.930 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:16:13.930 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:13 smithi078 podman[100765]: 2024-05-09 04:16:13.583936777 +0000 UTC m=+0.667599403 container died b8496c5eb403604fe12315f5fc710a505f1e82cc71b67398e10af29c21e93221 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240507, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux ) 2024-05-09T04:16:14.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:13 smithi132 ceph-mon[66347]: osdmap e120: 8 total, 7 up, 8 in 2024-05-09T04:16:14.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:14.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:16:14.511 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:14 smithi078 podman[100790]: 2024-05-09 04:16:14.213417029 +0000 UTC m=+0.623755573 container remove b8496c5eb403604fe12315f5fc710a505f1e82cc71b67398e10af29c21e93221 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:16:14.511 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:14 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.1.service: Deactivated successfully. 2024-05-09T04:16:14.512 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:14 smithi078 systemd[1]: Stopped Ceph osd.1 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:16:14.512 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:14 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.1.service: Consumed 14.913s CPU time. 2024-05-09T04:16:14.512 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:14 smithi078 systemd[1]: Starting Ceph osd.1 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:16:15.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:14 smithi132 ceph-mon[66347]: pgmap v85: 161 pgs: 19 stale+active+clean, 142 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-09T04:16:15.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:14 smithi078 ceph-mon[83176]: pgmap v85: 161 pgs: 19 stale+active+clean, 142 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-09T04:16:15.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:14 smithi078 ceph-mon[88406]: pgmap v85: 161 pgs: 19 stale+active+clean, 142 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-09T04:16:15.011 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:14 smithi078 podman[100892]: 2024-05-09 04:16:14.664206696 +0000 UTC m=+0.113333824 container create 8d1b3721dae0716e90e51e93e2f4179cfce36281719fed81badfbf13ee6f33fd (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:16:15.011 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:14 smithi078 podman[100892]: 2024-05-09 04:16:14.569229818 +0000 UTC m=+0.018356967 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:16:15.012 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:14 smithi078 podman[100892]: 2024-05-09 04:16:14.909411369 +0000 UTC m=+0.358538506 container init 8d1b3721dae0716e90e51e93e2f4179cfce36281719fed81badfbf13ee6f33fd (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-05-09T04:16:15.012 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:14 smithi078 podman[100892]: 2024-05-09 04:16:14.912873281 +0000 UTC m=+0.362000423 container start 8d1b3721dae0716e90e51e93e2f4179cfce36281719fed81badfbf13ee6f33fd (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:16:15.012 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 podman[100892]: 2024-05-09 04:16:15.008996871 +0000 UTC m=+0.458124002 container attach 8d1b3721dae0716e90e51e93e2f4179cfce36281719fed81badfbf13ee6f33fd (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-05-09T04:16:15.704 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate[100907]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:16:15.704 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate[100907]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:16:15.704 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 bash[100892]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:16:15.704 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 bash[100892]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:16:15.704 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 bash[100892]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-09T04:16:15.704 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 bash[100892]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-09T04:16:15.704 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 bash[100892]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-09T04:16:15.704 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 bash[100892]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-09T04:16:15.704 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 bash[100892]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-05-09T04:16:15.704 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 bash[100892]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-09T04:16:15.704 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 bash[100892]: --> ceph-volume raw activate successful for osd ID: 1 2024-05-09T04:16:15.704 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate[100907]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-09T04:16:15.705 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate[100907]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-09T04:16:15.705 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate[100907]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-09T04:16:15.705 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate[100907]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-09T04:16:15.705 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate[100907]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-05-09T04:16:15.705 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate[100907]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-09T04:16:15.705 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate[100907]: --> ceph-volume raw activate successful for osd ID: 1 2024-05-09T04:16:15.705 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:15 smithi078 podman[100892]: 2024-05-09 04:16:15.422298173 +0000 UTC m=+0.871425293 container died 8d1b3721dae0716e90e51e93e2f4179cfce36281719fed81badfbf13ee6f33fd (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-05-09T04:16:16.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:15 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:16.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:15 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:16.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:15 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:16.375 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:16 smithi078 podman[101043]: 2024-05-09 04:16:16.029312912 +0000 UTC m=+0.601529162 container remove 8d1b3721dae0716e90e51e93e2f4179cfce36281719fed81badfbf13ee6f33fd (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, io.buildah.version=1.33.5) 2024-05-09T04:16:16.375 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:16 smithi078 podman[101108]: 2024-05-09 04:16:16.146614987 +0000 UTC m=+0.018295748 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:16:16.376 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:16 smithi078 podman[101108]: 2024-05-09 04:16:16.271692997 +0000 UTC m=+0.143373758 container create c1e61031cb25a21eef32e2e15dd1b45140ac70deb478435d14c7297c8ad7242d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:16:16.733 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:16 smithi078 podman[101108]: 2024-05-09 04:16:16.482005154 +0000 UTC m=+0.353685915 container init c1e61031cb25a21eef32e2e15dd1b45140ac70deb478435d14c7297c8ad7242d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, ceph=True) 2024-05-09T04:16:16.733 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:16 smithi078 podman[101108]: 2024-05-09 04:16:16.48601788 +0000 UTC m=+0.357698643 container start c1e61031cb25a21eef32e2e15dd1b45140ac70deb478435d14c7297c8ad7242d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-05-09T04:16:16.733 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:16 smithi078 ceph-osd[101126]: -- 172.21.15.78:0/3309270708 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55dd57dc1680 con 0x55dd57d8b000 2024-05-09T04:16:16.733 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:16 smithi078 bash[101108]: c1e61031cb25a21eef32e2e15dd1b45140ac70deb478435d14c7297c8ad7242d 2024-05-09T04:16:16.733 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:16 smithi078 systemd[1]: Started Ceph osd.1 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:16:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:16 smithi132 ceph-mon[66347]: pgmap v86: 161 pgs: 18 active+undersized+degraded, 1 stale+active+clean, 38 active+undersized, 104 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-05-09T04:16:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:16 smithi132 ceph-mon[66347]: Health check failed: Degraded data redundancy: 93/723 objects degraded (12.863%), 18 pgs degraded (PG_DEGRADED) 2024-05-09T04:16:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:17.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:17.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:16 smithi078 ceph-mon[83176]: pgmap v86: 161 pgs: 18 active+undersized+degraded, 1 stale+active+clean, 38 active+undersized, 104 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-05-09T04:16:17.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:16 smithi078 ceph-mon[83176]: Health check failed: Degraded data redundancy: 93/723 objects degraded (12.863%), 18 pgs degraded (PG_DEGRADED) 2024-05-09T04:16:17.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:17.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:17.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:17.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:17.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:16 smithi078 ceph-mon[88406]: pgmap v86: 161 pgs: 18 active+undersized+degraded, 1 stale+active+clean, 38 active+undersized, 104 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-05-09T04:16:17.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:16 smithi078 ceph-mon[88406]: Health check failed: Degraded data redundancy: 93/723 objects degraded (12.863%), 18 pgs degraded (PG_DEGRADED) 2024-05-09T04:16:17.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:17.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:17.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:17.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:17.403 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:17 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1[101122]: 2024-05-09T04:16:17.027+0000 7f59d88867c0 -1 Falling back to public interface 2024-05-09T04:16:19.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:18 smithi132 ceph-mon[66347]: pgmap v87: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 99/723 objects degraded (13.693%) 2024-05-09T04:16:19.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:18 smithi078 ceph-mon[88406]: pgmap v87: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 99/723 objects degraded (13.693%) 2024-05-09T04:16:19.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:18 smithi078 ceph-mon[83176]: pgmap v87: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 99/723 objects degraded (13.693%) 2024-05-09T04:16:19.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:16:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:16:19] "GET /metrics HTTP/1.1" 200 38249 "" "Prometheus/2.43.0" 2024-05-09T04:16:20.011 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1[101122]: 2024-05-09T04:16:19.748+0000 7f59d88867c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-05-09T04:16:20.011 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1[101122]: 2024-05-09T04:16:19.781+0000 7f59d88867c0 -1 osd.1 118 log_to_monitors true 2024-05-09T04:16:20.749 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:20 smithi078 ceph-mon[83176]: pgmap v88: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 99/723 objects degraded (13.693%) 2024-05-09T04:16:20.749 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:20 smithi078 ceph-mon[83176]: from='osd.1 [v2:172.21.15.78:6810/4059195075,v1:172.21.15.78:6811/4059195075]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-09T04:16:20.750 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:20 smithi078 ceph-mon[83176]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-09T04:16:21.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:20 smithi132 ceph-mon[66347]: pgmap v88: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 99/723 objects degraded (13.693%) 2024-05-09T04:16:21.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:20 smithi132 ceph-mon[66347]: from='osd.1 [v2:172.21.15.78:6810/4059195075,v1:172.21.15.78:6811/4059195075]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-09T04:16:21.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:20 smithi132 ceph-mon[66347]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-09T04:16:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:20 smithi078 ceph-mon[88406]: pgmap v88: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 99/723 objects degraded (13.693%) 2024-05-09T04:16:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:20 smithi078 ceph-mon[88406]: from='osd.1 [v2:172.21.15.78:6810/4059195075,v1:172.21.15.78:6811/4059195075]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-09T04:16:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:20 smithi078 ceph-mon[88406]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-09T04:16:21.011 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:16:20 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1[101122]: 2024-05-09T04:16:20.773+0000 7f59c9cdd700 -1 osd.1 118 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T04:16:22.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:21 smithi078 ceph-mon[88406]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-09T04:16:22.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:21 smithi078 ceph-mon[88406]: osdmap e121: 8 total, 7 up, 8 in 2024-05-09T04:16:22.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:21 smithi078 ceph-mon[88406]: from='osd.1 [v2:172.21.15.78:6810/4059195075,v1:172.21.15.78:6811/4059195075]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:16:22.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:21 smithi078 ceph-mon[88406]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:16:22.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:21 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:22.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:21 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:22.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:21 smithi078 ceph-mon[83176]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-09T04:16:22.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:21 smithi078 ceph-mon[83176]: osdmap e121: 8 total, 7 up, 8 in 2024-05-09T04:16:22.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:21 smithi078 ceph-mon[83176]: from='osd.1 [v2:172.21.15.78:6810/4059195075,v1:172.21.15.78:6811/4059195075]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:16:22.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:21 smithi078 ceph-mon[83176]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:16:22.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:21 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:22.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:21 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:22.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:21 smithi132 ceph-mon[66347]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-09T04:16:22.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:21 smithi132 ceph-mon[66347]: osdmap e121: 8 total, 7 up, 8 in 2024-05-09T04:16:22.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:21 smithi132 ceph-mon[66347]: from='osd.1 [v2:172.21.15.78:6810/4059195075,v1:172.21.15.78:6811/4059195075]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:16:22.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:21 smithi132 ceph-mon[66347]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:16:22.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:21 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:22.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:21 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:22.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[83176]: pgmap v90: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 99/723 objects degraded (13.693%) 2024-05-09T04:16:22.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[83176]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:16:22.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[83176]: osd.1 [v2:172.21.15.78:6810/4059195075,v1:172.21.15.78:6811/4059195075] boot 2024-05-09T04:16:22.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[83176]: osdmap e122: 8 total, 8 up, 8 in 2024-05-09T04:16:22.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:16:22.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:22.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:22.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[83176]: Health check update: Degraded data redundancy: 99/723 objects degraded (13.693%), 19 pgs degraded (PG_DEGRADED) 2024-05-09T04:16:22.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[88406]: pgmap v90: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 99/723 objects degraded (13.693%) 2024-05-09T04:16:22.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[88406]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:16:22.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[88406]: osd.1 [v2:172.21.15.78:6810/4059195075,v1:172.21.15.78:6811/4059195075] boot 2024-05-09T04:16:22.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[88406]: osdmap e122: 8 total, 8 up, 8 in 2024-05-09T04:16:22.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:16:22.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:22.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:22.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:22 smithi078 ceph-mon[88406]: Health check update: Degraded data redundancy: 99/723 objects degraded (13.693%), 19 pgs degraded (PG_DEGRADED) 2024-05-09T04:16:23.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:22 smithi132 ceph-mon[66347]: pgmap v90: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 99/723 objects degraded (13.693%) 2024-05-09T04:16:23.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:22 smithi132 ceph-mon[66347]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:16:23.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:22 smithi132 ceph-mon[66347]: osd.1 [v2:172.21.15.78:6810/4059195075,v1:172.21.15.78:6811/4059195075] boot 2024-05-09T04:16:23.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:22 smithi132 ceph-mon[66347]: osdmap e122: 8 total, 8 up, 8 in 2024-05-09T04:16:23.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:22 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-09T04:16:23.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:22 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:23.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:22 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:23.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:22 smithi132 ceph-mon[66347]: Health check update: Degraded data redundancy: 99/723 objects degraded (13.693%), 19 pgs degraded (PG_DEGRADED) 2024-05-09T04:16:24.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:23 smithi078 ceph-mon[83176]: osdmap e123: 8 total, 8 up, 8 in 2024-05-09T04:16:24.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:23 smithi078 ceph-mon[88406]: osdmap e123: 8 total, 8 up, 8 in 2024-05-09T04:16:24.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:23 smithi132 ceph-mon[66347]: osdmap e123: 8 total, 8 up, 8 in 2024-05-09T04:16:25.204 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:24 smithi078 ceph-mon[83176]: pgmap v93: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 99/723 objects degraded (13.693%) 2024-05-09T04:16:25.204 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:24 smithi078 ceph-mon[88406]: pgmap v93: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 99/723 objects degraded (13.693%) 2024-05-09T04:16:25.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:24 smithi132 ceph-mon[66347]: pgmap v93: 161 pgs: 19 active+undersized+degraded, 44 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 99/723 objects degraded (13.693%) 2024-05-09T04:16:26.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:25 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:26.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:26.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:26.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:26.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:16:26.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:26.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:25 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:26.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:26.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:26.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:26.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:16:26.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:26.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:25 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:26.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:16:26.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:27.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: pgmap v94: 161 pgs: 6 active+undersized+degraded, 19 active+undersized, 136 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 43/723 objects degraded (5.947%) 2024-05-09T04:16:27.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:27.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:16:27.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:27.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: Upgrade: Finalizing container_image settings 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:16:27.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: Upgrade: Complete! 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:16:27.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:27.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:27.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:16:27.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:27.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:27.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:27.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:16:27.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:26 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:27.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: pgmap v94: 161 pgs: 6 active+undersized+degraded, 19 active+undersized, 136 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 43/723 objects degraded (5.947%) 2024-05-09T04:16:27.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:27.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:16:27.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:27.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: Upgrade: Finalizing container_image settings 2024-05-09T04:16:27.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:16:27.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:16:27.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:16:27.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:16:27.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:16:27.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:16:27.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:16:27.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:16:27.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:16:27.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:16:27.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: pgmap v94: 161 pgs: 6 active+undersized+degraded, 19 active+undersized, 136 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 43/723 objects degraded (5.947%) 2024-05-09T04:16:27.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:27.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:16:27.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:27.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.264 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: Upgrade: Finalizing container_image settings 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:16:27.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: Upgrade: Complete! 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:16:27.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.267 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: Upgrade: Complete! 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:16:27.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:26 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:28.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:27 smithi132 ceph-mon[66347]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-05-09T04:16:28.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:27 smithi132 ceph-mon[66347]: Health check update: Degraded data redundancy: 43/723 objects degraded (5.947%), 6 pgs degraded (PG_DEGRADED) 2024-05-09T04:16:28.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:27 smithi078 ceph-mon[83176]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-05-09T04:16:28.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:27 smithi078 ceph-mon[83176]: Health check update: Degraded data redundancy: 43/723 objects degraded (5.947%), 6 pgs degraded (PG_DEGRADED) 2024-05-09T04:16:28.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:27 smithi078 ceph-mon[88406]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-05-09T04:16:28.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:27 smithi078 ceph-mon[88406]: Health check update: Degraded data redundancy: 43/723 objects degraded (5.947%), 6 pgs degraded (PG_DEGRADED) 2024-05-09T04:16:29.058 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:28 smithi078 ceph-mon[83176]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/723 objects degraded (5.947%), 6 pgs degraded) 2024-05-09T04:16:29.058 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:28 smithi078 ceph-mon[83176]: Cluster is now healthy 2024-05-09T04:16:29.058 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:29.059 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:16:29.059 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:29.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:28 smithi078 ceph-mon[88406]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/723 objects degraded (5.947%), 6 pgs degraded) 2024-05-09T04:16:29.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:28 smithi078 ceph-mon[88406]: Cluster is now healthy 2024-05-09T04:16:29.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:29.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:16:29.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:29.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:28 smithi132 ceph-mon[66347]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/723 objects degraded (5.947%), 6 pgs degraded) 2024-05-09T04:16:29.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:28 smithi132 ceph-mon[66347]: Cluster is now healthy 2024-05-09T04:16:29.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:29.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:16:29.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:29.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:16:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:16:29] "GET /metrics HTTP/1.1" 200 38254 "" "Prometheus/2.43.0" 2024-05-09T04:16:30.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:29 smithi132 ceph-mon[66347]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-05-09T04:16:30.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:29 smithi078 ceph-mon[83176]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-05-09T04:16:30.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:29 smithi078 ceph-mon[88406]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-05-09T04:16:32.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:31 smithi132 ceph-mon[66347]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 op/s 2024-05-09T04:16:32.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:31 smithi078 ceph-mon[83176]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 op/s 2024-05-09T04:16:32.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:31 smithi078 ceph-mon[88406]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 op/s 2024-05-09T04:16:33.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:32 smithi132 ceph-mon[66347]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:33.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:32 smithi078 ceph-mon[83176]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:33.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:32 smithi078 ceph-mon[88406]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:33.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T04:16:34.036 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:34 smithi078 ceph-mon[88406]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-09T04:16:34.037 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:34.037 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:34 smithi078 ceph-mon[83176]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-09T04:16:34.037 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:34.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:34 smithi132 ceph-mon[66347]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-09T04:16:34.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:35.134 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:35 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:35.135 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:35 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:35.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:35 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:35.597 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:16:35.597 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (5m) 14s ago 27m 18.4M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:16:35.597 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (4m) 2m ago 26m 80.8M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:16:35.597 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (6m) 14s ago 26m 64.9M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:16:35.597 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (6m) 2m ago 31m 463M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:16:35.597 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (16m) 14s ago 33m 521M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:16:35.597 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (3m) 14s ago 33m 47.8M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:16:35.597 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (3m) 2m ago 32m 36.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:16:35.597 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (2m) 14s ago 32m 36.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (6m) 14s ago 27m 16.5M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (5m) 2m ago 27m 17.3M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (54s) 14s ago 31m 46.0M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (19s) 14s ago 31m 14.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (30m) 14s ago 30m 53.0M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (30m) 14s ago 30m 56.6M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (29m) 2m ago 29m 58.2M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (29m) 2m ago 29m 52.7M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (29m) 2m ago 29m 53.5M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (28m) 2m ago 28m 57.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (5m) 2m ago 28m 65.5M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (26m) 14s ago 26m 79.7M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:16:35.598 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (26m) 2m ago 26m 80.6M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:16:36.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:36 smithi078 ceph-mon[83176]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-05-09T04:16:36.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:36 smithi078 ceph-mon[83176]: from='client.44322 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:36.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:36 smithi078 ceph-mon[88406]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-05-09T04:16:36.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:36 smithi078 ceph-mon[88406]: from='client.44322 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:36.388 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-05-09T04:16:36.404 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:36 smithi132 ceph-mon[66347]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-05-09T04:16:36.404 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:36 smithi132 ceph-mon[66347]: from='client.44322 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:38.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:38 smithi078 ceph-mon[83176]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-09T04:16:38.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:38 smithi078 ceph-mon[88406]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-09T04:16:38.507 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:38 smithi132 ceph-mon[66347]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-09T04:16:38.527 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:16:39.053 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:39 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/3744658580' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:39.054 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:39 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/3744658580' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:39.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-05-09T04:16:39.382 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:16:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:16:39] "GET /metrics HTTP/1.1" 200 38254 "" "Prometheus/2.43.0" 2024-05-09T04:16:39.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:39 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/3744658580' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:40.476 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:40 smithi078 ceph-mon[88406]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:40.477 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:40 smithi078 ceph-mon[83176]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:40.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:40 smithi132 ceph-mon[66347]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:42.408 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:42 smithi078 ceph-mon[88406]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:42.409 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:42 smithi078 ceph-mon[88406]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:42.409 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:42 smithi078 ceph-mon[83176]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:42.409 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:42 smithi078 ceph-mon[83176]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:42.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:42 smithi132 ceph-mon[66347]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:42.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:42 smithi132 ceph-mon[66347]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:43.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:16:43.356 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:16:43.423 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:16:43.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:16:44.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade status' 2024-05-09T04:16:44.402 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:44 smithi078 ceph-mon[83176]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:44.402 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:44 smithi078 ceph-mon[88406]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:44.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:44 smithi132 ceph-mon[66347]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:45.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:45 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:45.350 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:45 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:45.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:45 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:46.059 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:16:46.059 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": null, 2024-05-09T04:16:46.059 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": false, 2024-05-09T04:16:46.059 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "", 2024-05-09T04:16:46.059 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:16:46.060 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": null, 2024-05-09T04:16:46.060 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-05-09T04:16:46.060 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:16:46.060 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:16:46.389 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:46 smithi078 ceph-mon[88406]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:46.389 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:46 smithi078 ceph-mon[83176]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:46.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:46 smithi132 ceph-mon[66347]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:46.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph health detail' 2024-05-09T04:16:47.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:47 smithi078 ceph-mon[83176]: from='client.54277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:47.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:47 smithi078 ceph-mon[88406]: from='client.54277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:47.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:47 smithi132 ceph-mon[66347]: from='client.54277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:48.237 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:48 smithi078 ceph-mon[88406]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:48.238 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:48 smithi078 ceph-mon[83176]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:48.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:48 smithi132 ceph-mon[66347]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:48.748 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-05-09T04:16:49.409 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:49 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/1866096016' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:16:49.409 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:16:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:16:49] "GET /metrics HTTP/1.1" 200 38255 "" "Prometheus/2.43.0" 2024-05-09T04:16:49.409 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:49 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/1866096016' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:16:49.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-05-09T04:16:49.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:49 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/1866096016' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:16:50.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:50 smithi078 ceph-mon[88406]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:50.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:50 smithi078 ceph-mon[83176]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:50.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:50 smithi132 ceph-mon[66347]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:52.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:52 smithi132 ceph-mon[66347]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:52.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:52 smithi132 ceph-mon[66347]: from='client.44352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:52.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:52 smithi078 ceph-mon[83176]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:52.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:52 smithi078 ceph-mon[83176]: from='client.44352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:52.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:52 smithi078 ceph-mon[88406]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:52.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:52 smithi078 ceph-mon[88406]: from='client.44352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:54.366 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:54 smithi078 ceph-mon[88406]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:54.366 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:54 smithi078 ceph-mon[83176]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:54.412 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:16:54.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:54 smithi132 ceph-mon[66347]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:55.012 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-09T04:16:55.598 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:55.598 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[83176]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:16:55.598 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:55.599 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:55.599 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:55.599 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:16:55.599 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:55.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:55.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[88406]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:16:55.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:55.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:55.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:55.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:16:55.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:55 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:55.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:55 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:16:55.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:55 smithi132 ceph-mon[66347]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:16:55.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:55 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:55.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:55 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:55 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:16:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:55 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:16:55.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:55 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:56.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:56 smithi078 ceph-mon[83176]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:56.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:56 smithi078 ceph-mon[83176]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:16:56.628 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:56 smithi078 ceph-mon[88406]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:56.628 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:56 smithi078 ceph-mon[88406]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:16:56.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:56 smithi132 ceph-mon[66347]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:16:56.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:56 smithi132 ceph-mon[66347]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:16:57.079 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:16:57.771 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:16:57.771 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (5m) 36s ago 27m 18.4M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (5m) 3m ago 27m 80.8M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (6m) 36s ago 26m 64.9M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (6m) 3m ago 32m 463M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (16m) 36s ago 33m 521M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (3m) 36s ago 33m 47.8M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (4m) 3m ago 32m 36.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (2m) 36s ago 32m 36.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (6m) 36s ago 28m 16.5M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (6m) 3m ago 28m 17.3M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (76s) 36s ago 31m 46.0M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (41s) 36s ago 31m 14.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (31m) 36s ago 31m 53.0M 2992M 17.2.0 e1d6a67b021e eee97e228b69 2024-05-09T04:16:57.772 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (30m) 36s ago 30m 56.6M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:16:57.773 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (30m) 3m ago 30m 58.2M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:16:57.773 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (29m) 3m ago 29m 52.7M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:16:57.773 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (29m) 3m ago 29m 53.5M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:16:57.773 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (29m) 3m ago 29m 57.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:16:57.773 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (5m) 3m ago 28m 65.5M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:16:57.773 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (26m) 36s ago 26m 79.7M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:16:57.773 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (26m) 3m ago 26m 80.6M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:16:58.190 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:16:58.190 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:16:58.190 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:16:58.190 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:16:58.190 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:16:58.190 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:16:58.190 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:16:58.190 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:16:58.190 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-05-09T04:16:58.190 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:16:58.190 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:16:58.190 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:16:58.191 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:16:58.191 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:16:58.191 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:16:58.191 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-05-09T04:16:58.191 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 7 2024-05-09T04:16:58.191 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:16:58.191 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:16:58.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:58.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='client.54295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:58.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:16:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/3546968337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:58.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='client.54295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/3546968337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:58.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:58.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:58.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-09T04:16:58.565 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:16:58.565 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:16:58.565 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:16:58.565 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-05-09T04:16:58.565 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:16:58.565 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/8 daemons upgraded", 2024-05-09T04:16:58.565 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-05-09T04:16:58.566 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:16:58.566 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:16:58.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:16:58.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='client.54295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:58.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:58.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:58.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:16:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/3546968337' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:16:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:16:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:16:58.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-09T04:16:59.228 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:16:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:16:59] "GET /metrics HTTP/1.1" 200 38255 "" "Prometheus/2.43.0" 2024-05-09T04:16:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[83176]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:16:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[83176]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:16:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mgr 2024-05-09T04:16:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mon 2024-05-09T04:16:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all crash 2024-05-09T04:16:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-09T04:16:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[83176]: Upgrade: osd.2 is safe to restart 2024-05-09T04:16:59.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[83176]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[88406]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:16:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[88406]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:16:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mgr 2024-05-09T04:16:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mon 2024-05-09T04:16:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all crash 2024-05-09T04:16:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-09T04:16:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[88406]: Upgrade: osd.2 is safe to restart 2024-05-09T04:16:59.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:16:59 smithi078 ceph-mon[88406]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:16:59.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:59 smithi132 ceph-mon[66347]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:16:59.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:59 smithi132 ceph-mon[66347]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:16:59.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:59 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mgr 2024-05-09T04:16:59.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:59 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mon 2024-05-09T04:16:59.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:59 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all crash 2024-05-09T04:16:59.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:59 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-09T04:16:59.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:59 smithi132 ceph-mon[66347]: Upgrade: osd.2 is safe to restart 2024-05-09T04:16:59.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:16:59 smithi132 ceph-mon[66347]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:00.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:00 smithi132 ceph-mon[66347]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:00.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:00 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:00.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:00 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-09T04:17:00.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:00 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:00.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:00 smithi078 ceph-mon[88406]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:00.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:00 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:00.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:00 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-09T04:17:00.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:00 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:00.759 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:00 smithi078 ceph-mon[83176]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:00.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:00 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:00.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:00 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-09T04:17:00.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:00 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:01.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:01 smithi078 ceph-mon[88406]: Upgrade: Updating osd.2 2024-05-09T04:17:01.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:01 smithi078 ceph-mon[88406]: Deploying daemon osd.2 on smithi078 2024-05-09T04:17:01.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:01 smithi078 ceph-mon[83176]: Upgrade: Updating osd.2 2024-05-09T04:17:01.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:01 smithi078 ceph-mon[83176]: Deploying daemon osd.2 on smithi078 2024-05-09T04:17:01.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:01 smithi132 ceph-mon[66347]: Upgrade: Updating osd.2 2024-05-09T04:17:01.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:01 smithi132 ceph-mon[66347]: Deploying daemon osd.2 on smithi078 2024-05-09T04:17:02.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:02 smithi132 ceph-mon[66347]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:02.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:02 smithi078 ceph-mon[88406]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:02.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:02 smithi078 ceph-mon[83176]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:03.452 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:03 smithi078 systemd[1]: Stopping Ceph osd.2 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:17:03.452 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2[36375]: 2024-05-09T04:17:03.416+0000 7ffb37448700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:17:03.452 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2[36375]: 2024-05-09T04:17:03.416+0000 7ffb37448700 -1 osd.2 123 *** Got signal Terminated *** 2024-05-09T04:17:03.452 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2[36375]: 2024-05-09T04:17:03.416+0000 7ffb37448700 -1 osd.2 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:17:03.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:03 smithi132 ceph-mon[66347]: osd.2 marked itself down and dead 2024-05-09T04:17:03.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:03 smithi078 ceph-mon[83176]: osd.2 marked itself down and dead 2024-05-09T04:17:03.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:03 smithi078 ceph-mon[88406]: osd.2 marked itself down and dead 2024-05-09T04:17:04.060 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:03 smithi078 podman[107134]: 2024-05-09 04:17:03.776908211 +0000 UTC m=+0.460067068 container died eee97e228b699028db8b4ef77918e02c62415ad3d4a385676b090104701cf7ea (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, RELEASE=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, release=754, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, vcs-type=git, ceph=True, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8) 2024-05-09T04:17:04.401 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:04 smithi078 podman[107134]: 2024-05-09 04:17:04.158708536 +0000 UTC m=+0.841867385 container cleanup eee97e228b699028db8b4ef77918e02c62415ad3d4a385676b090104701cf7ea (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2, GIT_BRANCH=HEAD, ceph=True, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, version=8, maintainer=Guillaume Abrioux , architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-09T04:17:04.401 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:04 smithi078 bash[107134]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2 2024-05-09T04:17:04.741 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:04 smithi078 ceph-mon[88406]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:04.742 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:04 smithi078 ceph-mon[88406]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:17:04.742 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:04 smithi078 ceph-mon[88406]: osdmap e124: 8 total, 7 up, 8 in 2024-05-09T04:17:04.743 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:04 smithi078 ceph-mon[83176]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:04.743 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:04 smithi078 ceph-mon[83176]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:17:04.743 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:04 smithi078 ceph-mon[83176]: osdmap e124: 8 total, 7 up, 8 in 2024-05-09T04:17:04.743 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:04 smithi078 podman[107151]: 2024-05-09 04:17:04.401623686 +0000 UTC m=+0.619232506 container remove eee97e228b699028db8b4ef77918e02c62415ad3d4a385676b090104701cf7ea (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2, io.openshift.tags=base centos centos-stream, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, ceph=True, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, version=8, vcs-type=git, com.redhat.component=centos-stream-container, io.openshift.expose-services=, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc.) 2024-05-09T04:17:04.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:04 smithi132 ceph-mon[66347]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:04.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:04 smithi132 ceph-mon[66347]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:17:04.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:04 smithi132 ceph-mon[66347]: osdmap e124: 8 total, 7 up, 8 in 2024-05-09T04:17:05.011 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:04 smithi078 podman[107241]: 2024-05-09 04:17:04.64362927 +0000 UTC m=+0.018464359 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:17:05.011 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:04 smithi078 podman[107241]: 2024-05-09 04:17:04.769287996 +0000 UTC m=+0.144123100 container create 392561dae586cf09124f7b610a201b049264eb9848d68e7580ac33a4982482a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-05-09T04:17:05.012 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:04 smithi078 podman[107241]: 2024-05-09 04:17:04.980515347 +0000 UTC m=+0.355350409 container init 392561dae586cf09124f7b610a201b049264eb9848d68e7580ac33a4982482a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-deactivate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.build-date=20240507, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-05-09T04:17:05.012 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:04 smithi078 podman[107241]: 2024-05-09 04:17:04.984108872 +0000 UTC m=+0.358943944 container start 392561dae586cf09124f7b610a201b049264eb9848d68e7580ac33a4982482a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-deactivate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-05-09T04:17:05.464 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:05 smithi078 podman[107241]: 2024-05-09 04:17:05.063670492 +0000 UTC m=+0.438505564 container attach 392561dae586cf09124f7b610a201b049264eb9848d68e7580ac33a4982482a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240507, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-05-09T04:17:05.464 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:05 smithi078 podman[107241]: 2024-05-09 04:17:05.283869903 +0000 UTC m=+0.658704975 container died 392561dae586cf09124f7b610a201b049264eb9848d68e7580ac33a4982482a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2) 2024-05-09T04:17:05.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:05 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:05.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:05 smithi132 ceph-mon[66347]: osdmap e125: 8 total, 7 up, 8 in 2024-05-09T04:17:05.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:05 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:05.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:05 smithi078 ceph-mon[88406]: osdmap e125: 8 total, 7 up, 8 in 2024-05-09T04:17:05.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:05 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:05.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:05 smithi078 ceph-mon[83176]: osdmap e125: 8 total, 7 up, 8 in 2024-05-09T04:17:06.465 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:06 smithi078 podman[107267]: 2024-05-09 04:17:06.092224623 +0000 UTC m=+0.801478817 container remove 392561dae586cf09124f7b610a201b049264eb9848d68e7580ac33a4982482a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:17:06.465 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:06 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.2.service: Deactivated successfully. 2024-05-09T04:17:06.465 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:06 smithi078 systemd[1]: Stopped Ceph osd.2 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:17:06.465 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:06 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.2.service: Consumed 12.478s CPU time. 2024-05-09T04:17:06.465 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:06 smithi078 systemd[1]: Starting Ceph osd.2 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:17:06.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:06 smithi132 ceph-mon[66347]: pgmap v116: 161 pgs: 33 peering, 4 stale+active+clean, 124 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:17:06.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:06 smithi132 ceph-mon[66347]: Health check failed: Reduced data availability: 7 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-05-09T04:17:06.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:06 smithi078 ceph-mon[83176]: pgmap v116: 161 pgs: 33 peering, 4 stale+active+clean, 124 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:17:06.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:06 smithi078 ceph-mon[83176]: Health check failed: Reduced data availability: 7 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-05-09T04:17:06.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:06 smithi078 ceph-mon[88406]: pgmap v116: 161 pgs: 33 peering, 4 stale+active+clean, 124 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:17:06.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:06 smithi078 ceph-mon[88406]: Health check failed: Reduced data availability: 7 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-05-09T04:17:06.761 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:06 smithi078 podman[107374]: 2024-05-09 04:17:06.442490748 +0000 UTC m=+0.018540823 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:17:06.762 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:06 smithi078 podman[107374]: 2024-05-09 04:17:06.568743864 +0000 UTC m=+0.144793982 container create b0eb19d765f93d0b52744c18ff5238f340987c10c4025aeddb88c1e97cfdcdc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:17:07.261 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:06 smithi078 podman[107374]: 2024-05-09 04:17:06.834309483 +0000 UTC m=+0.410359569 container init b0eb19d765f93d0b52744c18ff5238f340987c10c4025aeddb88c1e97cfdcdc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-05-09T04:17:07.261 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:06 smithi078 podman[107374]: 2024-05-09 04:17:06.837751739 +0000 UTC m=+0.413801821 container start b0eb19d765f93d0b52744c18ff5238f340987c10c4025aeddb88c1e97cfdcdc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-05-09T04:17:07.261 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:06 smithi078 podman[107374]: 2024-05-09 04:17:06.912980664 +0000 UTC m=+0.489030756 container attach b0eb19d765f93d0b52744c18ff5238f340987c10c4025aeddb88c1e97cfdcdc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-05-09T04:17:07.641 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:07 smithi078 ceph-mon[83176]: Health check failed: Degraded data redundancy: 29/723 objects degraded (4.011%), 4 pgs degraded (PG_DEGRADED) 2024-05-09T04:17:07.642 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:07 smithi078 ceph-mon[88406]: Health check failed: Degraded data redundancy: 29/723 objects degraded (4.011%), 4 pgs degraded (PG_DEGRADED) 2024-05-09T04:17:07.642 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate[107390]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:17:07.642 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate[107390]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:17:07.642 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 bash[107374]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:17:07.642 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 bash[107374]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:17:07.643 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 bash[107374]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-09T04:17:07.643 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 bash[107374]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-09T04:17:07.643 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 bash[107374]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-09T04:17:07.643 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 bash[107374]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-09T04:17:07.643 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 bash[107374]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-05-09T04:17:07.643 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 bash[107374]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-09T04:17:07.644 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 bash[107374]: --> ceph-volume raw activate successful for osd ID: 2 2024-05-09T04:17:07.644 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate[107390]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-09T04:17:07.644 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate[107390]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-09T04:17:07.644 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate[107390]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-09T04:17:07.644 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate[107390]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-09T04:17:07.644 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate[107390]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-05-09T04:17:07.645 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate[107390]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-09T04:17:07.645 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate[107390]: --> ceph-volume raw activate successful for osd ID: 2 2024-05-09T04:17:07.645 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:07 smithi078 podman[107374]: 2024-05-09 04:17:07.34913004 +0000 UTC m=+0.925180124 container died b0eb19d765f93d0b52744c18ff5238f340987c10c4025aeddb88c1e97cfdcdc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:17:08.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:07 smithi132 ceph-mon[66347]: Health check failed: Degraded data redundancy: 29/723 objects degraded (4.011%), 4 pgs degraded (PG_DEGRADED) 2024-05-09T04:17:08.390 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:08 smithi078 podman[107515]: 2024-05-09 04:17:08.084248125 +0000 UTC m=+0.729584858 container remove b0eb19d765f93d0b52744c18ff5238f340987c10c4025aeddb88c1e97cfdcdc4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-activate, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-05-09T04:17:08.390 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:08 smithi078 podman[107580]: 2024-05-09 04:17:08.293555743 +0000 UTC m=+0.107950462 container create 65617f5ef4aeaa461e48c86505e96b46c79fe18e284dc524d8bab0c4f3104483 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=) 2024-05-09T04:17:08.391 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:08 smithi078 podman[107580]: 2024-05-09 04:17:08.203741534 +0000 UTC m=+0.018136263 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:17:08.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:08 smithi078 ceph-mon[83176]: pgmap v117: 161 pgs: 4 active+undersized+degraded, 33 peering, 10 active+undersized, 114 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 29/723 objects degraded (4.011%) 2024-05-09T04:17:08.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:08 smithi078 ceph-mon[88406]: pgmap v117: 161 pgs: 4 active+undersized+degraded, 33 peering, 10 active+undersized, 114 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 29/723 objects degraded (4.011%) 2024-05-09T04:17:08.762 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:08 smithi078 podman[107580]: 2024-05-09 04:17:08.487580969 +0000 UTC m=+0.301975701 container init 65617f5ef4aeaa461e48c86505e96b46c79fe18e284dc524d8bab0c4f3104483 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-05-09T04:17:08.762 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:08 smithi078 podman[107580]: 2024-05-09 04:17:08.490771649 +0000 UTC m=+0.305166368 container start 65617f5ef4aeaa461e48c86505e96b46c79fe18e284dc524d8bab0c4f3104483 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-09T04:17:08.762 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:08 smithi078 ceph-osd[107599]: -- 172.21.15.78:0/3820494345 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55de52f19680 con 0x55de52ee3400 2024-05-09T04:17:08.762 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:08 smithi078 bash[107580]: 65617f5ef4aeaa461e48c86505e96b46c79fe18e284dc524d8bab0c4f3104483 2024-05-09T04:17:08.762 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:08 smithi078 systemd[1]: Started Ceph osd.2 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:17:09.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:08 smithi132 ceph-mon[66347]: pgmap v117: 161 pgs: 4 active+undersized+degraded, 33 peering, 10 active+undersized, 114 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 29/723 objects degraded (4.011%) 2024-05-09T04:17:09.409 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:17:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:17:09] "GET /metrics HTTP/1.1" 200 38255 "" "Prometheus/2.43.0" 2024-05-09T04:17:09.409 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2[107595]: 2024-05-09T04:17:09.038+0000 7ff80f3fd7c0 -1 Falling back to public interface 2024-05-09T04:17:09.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:09 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:09.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:09 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:09.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:09 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:09.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:09 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:09.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:09 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:09.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:09 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:09.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:09 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:09.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:09 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:10.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:09 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:10.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:09 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:10.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:09 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:10.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:09 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:11.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:10 smithi132 ceph-mon[66347]: pgmap v118: 161 pgs: 16 active+undersized+degraded, 31 active+undersized, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-09T04:17:11.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:10 smithi132 ceph-mon[66347]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 9 pgs peering) 2024-05-09T04:17:11.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:10 smithi078 ceph-mon[88406]: pgmap v118: 161 pgs: 16 active+undersized+degraded, 31 active+undersized, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-09T04:17:11.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:10 smithi078 ceph-mon[88406]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 9 pgs peering) 2024-05-09T04:17:11.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:10 smithi078 ceph-mon[83176]: pgmap v118: 161 pgs: 16 active+undersized+degraded, 31 active+undersized, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-09T04:17:11.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:10 smithi078 ceph-mon[83176]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 9 pgs peering) 2024-05-09T04:17:12.011 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:11 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2[107595]: 2024-05-09T04:17:11.743+0000 7ff80f3fd7c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-05-09T04:17:12.011 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:11 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2[107595]: 2024-05-09T04:17:11.764+0000 7ff80f3fd7c0 -1 osd.2 123 log_to_monitors true 2024-05-09T04:17:13.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:12 smithi132 ceph-mon[66347]: pgmap v119: 161 pgs: 16 active+undersized+degraded, 31 active+undersized, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-09T04:17:13.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:12 smithi132 ceph-mon[66347]: from='osd.2 [v2:172.21.15.78:6818/448373494,v1:172.21.15.78:6819/448373494]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-09T04:17:13.011 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:17:12 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2[107595]: 2024-05-09T04:17:12.651+0000 7ff800854700 -1 osd.2 123 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T04:17:13.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:12 smithi078 ceph-mon[83176]: pgmap v119: 161 pgs: 16 active+undersized+degraded, 31 active+undersized, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-09T04:17:13.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:12 smithi078 ceph-mon[83176]: from='osd.2 [v2:172.21.15.78:6818/448373494,v1:172.21.15.78:6819/448373494]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-09T04:17:13.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:12 smithi078 ceph-mon[88406]: pgmap v119: 161 pgs: 16 active+undersized+degraded, 31 active+undersized, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-09T04:17:13.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:12 smithi078 ceph-mon[88406]: from='osd.2 [v2:172.21.15.78:6818/448373494,v1:172.21.15.78:6819/448373494]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-09T04:17:14.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:13 smithi132 ceph-mon[66347]: from='osd.2 [v2:172.21.15.78:6818/448373494,v1:172.21.15.78:6819/448373494]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-09T04:17:14.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:13 smithi132 ceph-mon[66347]: osdmap e126: 8 total, 7 up, 8 in 2024-05-09T04:17:14.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:13 smithi132 ceph-mon[66347]: from='osd.2 [v2:172.21.15.78:6818/448373494,v1:172.21.15.78:6819/448373494]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:17:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:13 smithi132 ceph-mon[66347]: Health check update: Degraded data redundancy: 69/723 objects degraded (9.544%), 16 pgs degraded (PG_DEGRADED) 2024-05-09T04:17:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:17:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:14.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:14.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[88406]: from='osd.2 [v2:172.21.15.78:6818/448373494,v1:172.21.15.78:6819/448373494]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-09T04:17:14.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[88406]: osdmap e126: 8 total, 7 up, 8 in 2024-05-09T04:17:14.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[88406]: from='osd.2 [v2:172.21.15.78:6818/448373494,v1:172.21.15.78:6819/448373494]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:17:14.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[88406]: Health check update: Degraded data redundancy: 69/723 objects degraded (9.544%), 16 pgs degraded (PG_DEGRADED) 2024-05-09T04:17:14.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:14.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:17:14.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:14.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:14.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[83176]: from='osd.2 [v2:172.21.15.78:6818/448373494,v1:172.21.15.78:6819/448373494]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-09T04:17:14.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[83176]: osdmap e126: 8 total, 7 up, 8 in 2024-05-09T04:17:14.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[83176]: from='osd.2 [v2:172.21.15.78:6818/448373494,v1:172.21.15.78:6819/448373494]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:17:14.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[83176]: Health check update: Degraded data redundancy: 69/723 objects degraded (9.544%), 16 pgs degraded (PG_DEGRADED) 2024-05-09T04:17:14.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:14.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:17:14.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:14.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:14.782 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[83176]: pgmap v121: 161 pgs: 16 active+undersized+degraded, 31 active+undersized, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-09T04:17:14.782 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[83176]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:17:14.782 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[83176]: osd.2 [v2:172.21.15.78:6818/448373494,v1:172.21.15.78:6819/448373494] boot 2024-05-09T04:17:14.782 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[83176]: osdmap e127: 8 total, 8 up, 8 in 2024-05-09T04:17:14.782 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:17:14.782 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:14.782 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:14.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[88406]: pgmap v121: 161 pgs: 16 active+undersized+degraded, 31 active+undersized, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-09T04:17:14.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[88406]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:17:14.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[88406]: osd.2 [v2:172.21.15.78:6818/448373494,v1:172.21.15.78:6819/448373494] boot 2024-05-09T04:17:14.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[88406]: osdmap e127: 8 total, 8 up, 8 in 2024-05-09T04:17:14.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:17:14.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:14.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:14 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:15.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:14 smithi132 ceph-mon[66347]: pgmap v121: 161 pgs: 16 active+undersized+degraded, 31 active+undersized, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 603 B/s rd, 0 op/s; 69/723 objects degraded (9.544%) 2024-05-09T04:17:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:14 smithi132 ceph-mon[66347]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:17:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:14 smithi132 ceph-mon[66347]: osd.2 [v2:172.21.15.78:6818/448373494,v1:172.21.15.78:6819/448373494] boot 2024-05-09T04:17:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:14 smithi132 ceph-mon[66347]: osdmap e127: 8 total, 8 up, 8 in 2024-05-09T04:17:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:14 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-09T04:17:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:14 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:15.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:14 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:15.928 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:15 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:15.928 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:15 smithi078 ceph-mon[83176]: osdmap e128: 8 total, 8 up, 8 in 2024-05-09T04:17:15.928 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:15 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:15.929 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:15 smithi078 ceph-mon[88406]: osdmap e128: 8 total, 8 up, 8 in 2024-05-09T04:17:16.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:15 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:16.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:15 smithi132 ceph-mon[66347]: osdmap e128: 8 total, 8 up, 8 in 2024-05-09T04:17:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:16 smithi132 ceph-mon[66347]: pgmap v124: 161 pgs: 24 peering, 6 active+undersized+degraded, 17 active+undersized, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 31/723 objects degraded (4.288%) 2024-05-09T04:17:17.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:16 smithi078 ceph-mon[83176]: pgmap v124: 161 pgs: 24 peering, 6 active+undersized+degraded, 17 active+undersized, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 31/723 objects degraded (4.288%) 2024-05-09T04:17:17.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:16 smithi078 ceph-mon[88406]: pgmap v124: 161 pgs: 24 peering, 6 active+undersized+degraded, 17 active+undersized, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 31/723 objects degraded (4.288%) 2024-05-09T04:17:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: pgmap v125: 161 pgs: 24 peering, 2 active+undersized+degraded, 11 active+undersized, 124 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 2/723 objects degraded (0.277%) 2024-05-09T04:17:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: Health check update: Degraded data redundancy: 2/723 objects degraded (0.277%), 2 pgs degraded (PG_DEGRADED) 2024-05-09T04:17:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:17:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:17:19.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:17:19.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:17:19.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:19.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:19.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:17:19.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:19.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:19.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:17:19.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: pgmap v125: 161 pgs: 24 peering, 2 active+undersized+degraded, 11 active+undersized, 124 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 2/723 objects degraded (0.277%) 2024-05-09T04:17:19.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: Health check update: Degraded data redundancy: 2/723 objects degraded (0.277%), 2 pgs degraded (PG_DEGRADED) 2024-05-09T04:17:19.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:17:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:17:19.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:17:19.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:17:19.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:17:19.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:17:19.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:17:19.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:17:19.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:17:19.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: pgmap v125: 161 pgs: 24 peering, 2 active+undersized+degraded, 11 active+undersized, 124 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 2/723 objects degraded (0.277%) 2024-05-09T04:17:19.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: Health check update: Degraded data redundancy: 2/723 objects degraded (0.277%), 2 pgs degraded (PG_DEGRADED) 2024-05-09T04:17:19.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:19.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:17:19.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:17:19.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:19.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:19.018 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:17:19.018 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:17:19.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:17:19.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:17:19.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:17:19.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:17:19.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:17:19.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:17:19.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:19.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:17:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:17:19] "GET /metrics HTTP/1.1" 200 38112 "" "Prometheus/2.43.0" 2024-05-09T04:17:20.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:19 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:17:20.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:19 smithi132 ceph-mon[66347]: Upgrade: Finalizing container_image settings 2024-05-09T04:17:20.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:19 smithi132 ceph-mon[66347]: Upgrade: Complete! 2024-05-09T04:17:20.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:19 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:17:20.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:19 smithi078 ceph-mon[83176]: Upgrade: Finalizing container_image settings 2024-05-09T04:17:20.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:19 smithi078 ceph-mon[83176]: Upgrade: Complete! 2024-05-09T04:17:20.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:19 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:17:20.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:19 smithi078 ceph-mon[88406]: Upgrade: Finalizing container_image settings 2024-05-09T04:17:20.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:19 smithi078 ceph-mon[88406]: Upgrade: Complete! 2024-05-09T04:17:21.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:20 smithi132 ceph-mon[66347]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-05-09T04:17:21.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:20 smithi132 ceph-mon[66347]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/723 objects degraded (0.277%), 2 pgs degraded) 2024-05-09T04:17:21.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:20 smithi132 ceph-mon[66347]: Cluster is now healthy 2024-05-09T04:17:21.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:20 smithi078 ceph-mon[83176]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-05-09T04:17:21.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:20 smithi078 ceph-mon[83176]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/723 objects degraded (0.277%), 2 pgs degraded) 2024-05-09T04:17:21.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:20 smithi078 ceph-mon[83176]: Cluster is now healthy 2024-05-09T04:17:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:20 smithi078 ceph-mon[88406]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-05-09T04:17:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:20 smithi078 ceph-mon[88406]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/723 objects degraded (0.277%), 2 pgs degraded) 2024-05-09T04:17:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:20 smithi078 ceph-mon[88406]: Cluster is now healthy 2024-05-09T04:17:23.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:22 smithi132 ceph-mon[66347]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:23.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:22 smithi078 ceph-mon[83176]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:23.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:22 smithi078 ceph-mon[88406]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:24.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:24 smithi132 ceph-mon[66347]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-09T04:17:24.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:24 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:24.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:24 smithi078 ceph-mon[83176]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-09T04:17:24.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:24 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:24.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:24 smithi078 ceph-mon[88406]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-09T04:17:24.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:24 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:25.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:25 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:25.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:25 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:25.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:25 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:26.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:26 smithi132 ceph-mon[66347]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-05-09T04:17:26.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:26 smithi078 ceph-mon[83176]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-05-09T04:17:26.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:26 smithi078 ceph-mon[88406]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-05-09T04:17:28.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:28 smithi132 ceph-mon[66347]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:28.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:28.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:17:28.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:28 smithi078 ceph-mon[83176]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:17:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:28.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:28 smithi078 ceph-mon[88406]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:28.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:28.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:17:28.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:29.230 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:17:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:17:29] "GET /metrics HTTP/1.1" 200 38266 "" "Prometheus/2.43.0" 2024-05-09T04:17:29.649 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T04:17:30.213 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:30 smithi078 ceph-mon[88406]: from='client.54331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:30.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:30 smithi078 ceph-mon[88406]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:30.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:30 smithi078 ceph-mon[83176]: from='client.54331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:30.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:30 smithi078 ceph-mon[83176]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:30.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:30 smithi132 ceph-mon[66347]: from='client.54331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:30.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:30 smithi132 ceph-mon[66347]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:31.710 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:17:31.710 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (6m) 18s ago 28m 28.0M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:17:31.710 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (5m) 3m ago 27m 80.8M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:17:31.710 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (7m) 18s ago 27m 67.1M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:17:31.710 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (7m) 3m ago 32m 463M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:17:31.710 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (17m) 18s ago 34m 523M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:17:31.710 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (3m) 18s ago 34m 50.2M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:17:31.710 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (4m) 3m ago 32m 36.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:17:31.710 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (3m) 18s ago 33m 39.4M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:17:31.710 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (6m) 18s ago 28m 16.5M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:17:31.710 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (6m) 3m ago 28m 17.3M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:17:31.711 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (110s) 18s ago 32m 46.3M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:17:31.711 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (75s) 18s ago 32m 46.0M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:17:31.711 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (23s) 18s ago 31m 11.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:17:31.711 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (31m) 18s ago 31m 57.0M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:17:31.711 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (30m) 3m ago 30m 58.2M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:17:31.711 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (30m) 3m ago 30m 52.7M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:17:31.711 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (30m) 3m ago 30m 53.5M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:17:31.711 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (29m) 3m ago 29m 57.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:17:31.711 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (6m) 3m ago 29m 65.5M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:17:31.711 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (27m) 18s ago 27m 79.9M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:17:31.711 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (27m) 3m ago 27m 80.6M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:17:32.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:32 smithi132 ceph-mon[66347]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:32.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:32 smithi132 ceph-mon[66347]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:32.504 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:32 smithi078 ceph-mon[88406]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:32.504 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:32 smithi078 ceph-mon[88406]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:32.505 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:32 smithi078 ceph-mon[83176]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:32.505 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:32 smithi078 ceph-mon[83176]: from='client.44400 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:32.529 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-05-09T04:17:34.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:34 smithi078 ceph-mon[83176]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:34.104 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:34 smithi078 ceph-mon[88406]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:34.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:34 smithi132 ceph-mon[66347]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:34.673 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:17:35.403 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:35 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:35.403 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:35 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/3325829764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:35.403 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:35 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:35.403 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:35 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/3325829764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:35.428 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-05-09T04:17:35.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:35 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:35.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:35 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/3325829764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:36.404 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:36 smithi132 ceph-mon[66347]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:36.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:36 smithi078 ceph-mon[88406]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:36.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:36 smithi078 ceph-mon[83176]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:38.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:38 smithi132 ceph-mon[66347]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:38.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:38 smithi132 ceph-mon[66347]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:38.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:38 smithi078 ceph-mon[83176]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:38.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:38 smithi078 ceph-mon[83176]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:38.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:38 smithi078 ceph-mon[88406]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:38.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:38 smithi078 ceph-mon[88406]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:39.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:17:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:17:39] "GET /metrics HTTP/1.1" 200 38266 "" "Prometheus/2.43.0" 2024-05-09T04:17:39.724 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:17:40.326 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:40 smithi078 ceph-mon[88406]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:40.326 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:40 smithi078 ceph-mon[83176]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:40.352 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade status' 2024-05-09T04:17:40.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:40 smithi132 ceph-mon[66347]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:42.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:42 smithi078 ceph-mon[83176]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:42.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:42 smithi078 ceph-mon[88406]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:42.325 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:17:42.325 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": null, 2024-05-09T04:17:42.325 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": false, 2024-05-09T04:17:42.325 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "", 2024-05-09T04:17:42.325 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:17:42.325 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": null, 2024-05-09T04:17:42.325 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-05-09T04:17:42.325 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:17:42.325 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:17:42.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:42 smithi132 ceph-mon[66347]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:42.904 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph health detail' 2024-05-09T04:17:43.219 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:43 smithi078 ceph-mon[83176]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:43.219 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:17:43.219 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:43 smithi078 ceph-mon[88406]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:43.219 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:17:43.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:43 smithi132 ceph-mon[66347]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:43.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:17:44.423 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:44 smithi078 ceph-mon[83176]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:44.423 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:44 smithi078 ceph-mon[88406]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:44.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:44 smithi132 ceph-mon[66347]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:45.133 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-05-09T04:17:45.412 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:45 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:45.413 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:45 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:45.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:45 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:45.753 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-05-09T04:17:46.324 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:46 smithi078 ceph-mon[88406]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:46.324 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:46 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/2619355193' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:17:46.325 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:46 smithi078 ceph-mon[83176]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:46.325 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:46 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/2619355193' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:17:46.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:46 smithi132 ceph-mon[66347]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:46.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:46 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2619355193' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:17:48.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:48 smithi132 ceph-mon[66347]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:48.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:48 smithi078 ceph-mon[83176]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:48.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:48 smithi078 ceph-mon[88406]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:49.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:49 smithi132 ceph-mon[66347]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:49.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:49 smithi078 ceph-mon[83176]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:49.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:49 smithi078 ceph-mon[88406]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:49.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:17:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:17:49] "GET /metrics HTTP/1.1" 200 38262 "" "Prometheus/2.43.0" 2024-05-09T04:17:49.850 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:17:50.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:50 smithi078 ceph-mon[83176]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:50.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:50.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:50.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:50.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:17:50.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:50 smithi132 ceph-mon[66347]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:50.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:50.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:17:50.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:50.510 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:50.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:50 smithi078 ceph-mon[88406]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:50.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:50.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:50.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:50.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:17:50.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:50.544 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-09T04:17:51.389 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:51 smithi078 ceph-mon[83176]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:17:51.390 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:51 smithi078 ceph-mon[83176]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:17:51.390 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:51 smithi078 ceph-mon[88406]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:17:51.390 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:51 smithi078 ceph-mon[88406]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:17:51.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:51 smithi132 ceph-mon[66347]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:17:51.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:51 smithi132 ceph-mon[66347]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:17:52.188 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:52 smithi078 ceph-mon[83176]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:52.189 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:52 smithi078 ceph-mon[88406]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:52.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:52 smithi132 ceph-mon[66347]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:53.465 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:17:54.175 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:17:54.175 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (6m) 41s ago 28m 28.0M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:17:54.175 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (6m) 4m ago 28m 80.8M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (7m) 41s ago 27m 67.1M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (7m) 4m ago 33m 463M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (17m) 41s ago 34m 523M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (4m) 41s ago 34m 50.2M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (5m) 4m ago 33m 36.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (3m) 41s ago 33m 39.4M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (7m) 41s ago 29m 16.5M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (7m) 4m ago 29m 17.3M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (2m) 41s ago 32m 46.3M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (98s) 41s ago 32m 46.0M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (45s) 41s ago 32m 11.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (31m) 41s ago 31m 57.0M 2992M 17.2.0 e1d6a67b021e d2a1e43b69d3 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (31m) 4m ago 31m 58.2M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:17:54.176 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (30m) 4m ago 30m 52.7M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:17:54.177 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (30m) 4m ago 30m 53.5M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:17:54.177 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (30m) 4m ago 30m 57.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:17:54.177 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (6m) 4m ago 29m 65.5M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:17:54.177 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (27m) 41s ago 27m 79.9M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:17:54.177 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (27m) 4m ago 27m 80.6M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:17:54.601 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:17:54.601 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:17:54.601 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:17:54.601 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:17:54.601 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:17:54.601 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:17:54.601 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:17:54.601 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:17:54.601 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-05-09T04:17:54.601 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:17:54.602 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:17:54.602 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:17:54.602 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:17:54.602 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:17:54.602 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:17:54.602 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-05-09T04:17:54.602 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 8 2024-05-09T04:17:54.602 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:17:54.602 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:17:54.962 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:17:54.962 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:17:54.962 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:17:54.962 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-09T04:17:54.963 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:17:54.963 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "3/8 daemons upgraded", 2024-05-09T04:17:54.963 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-05-09T04:17:54.963 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:17:54.963 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:17:54.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:54.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: from='client.44436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:54.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:54.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:17:54.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mgr 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mon 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all crash 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: Upgrade: osd.3 is safe to restart 2024-05-09T04:17:54.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/975437285' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: from='client.44436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mgr 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mon 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:54.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:54.966 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all crash 2024-05-09T04:17:54.966 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:54.966 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-09T04:17:54.966 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-09T04:17:54.966 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: Upgrade: osd.3 is safe to restart 2024-05-09T04:17:54.966 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:54 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/975437285' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:55.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:55.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: from='client.44436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:55.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:55.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mgr 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mon 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all crash 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-09T04:17:55.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: Upgrade: osd.3 is safe to restart 2024-05-09T04:17:55.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:54 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/975437285' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:17:56.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:55 smithi132 ceph-mon[66347]: from='client.54379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:56.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:55 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:56.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:55 smithi132 ceph-mon[66347]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:56.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:55 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:56.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:55 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-09T04:17:56.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:55 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:56.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:55 smithi078 ceph-mon[88406]: from='client.54379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:56.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:55 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:56.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:55 smithi078 ceph-mon[88406]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:56.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:55 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:56.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:55 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-09T04:17:56.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:55 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:56.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:55 smithi078 ceph-mon[83176]: from='client.54379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:56.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:55 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:17:56.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:55 smithi078 ceph-mon[83176]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:56.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:55 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:17:56.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:55 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-09T04:17:56.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:55 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:17:57.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:56 smithi132 ceph-mon[66347]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:57.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:56 smithi132 ceph-mon[66347]: from='client.44460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:57.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:56 smithi132 ceph-mon[66347]: Upgrade: Updating osd.3 2024-05-09T04:17:57.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:56 smithi132 ceph-mon[66347]: Deploying daemon osd.3 on smithi078 2024-05-09T04:17:57.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:56 smithi078 ceph-mon[88406]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:57.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:56 smithi078 ceph-mon[88406]: from='client.44460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:57.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:56 smithi078 ceph-mon[88406]: Upgrade: Updating osd.3 2024-05-09T04:17:57.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:56 smithi078 ceph-mon[88406]: Deploying daemon osd.3 on smithi078 2024-05-09T04:17:57.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:56 smithi078 ceph-mon[83176]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:17:57.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:56 smithi078 ceph-mon[83176]: from='client.44460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:17:57.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:56 smithi078 ceph-mon[83176]: Upgrade: Updating osd.3 2024-05-09T04:17:57.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:56 smithi078 ceph-mon[83176]: Deploying daemon osd.3 on smithi078 2024-05-09T04:17:58.708 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:17:58 smithi078 systemd[1]: Stopping Ceph osd.3 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:17:59.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:58 smithi132 ceph-mon[66347]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:59.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:17:59.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:58 smithi078 ceph-mon[83176]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:59.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:17:59.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:58 smithi078 ceph-mon[88406]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:17:59.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:17:59.012 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:17:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3[39353]: 2024-05-09T04:17:58.809+0000 7f2c56df0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:17:59.012 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:17:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3[39353]: 2024-05-09T04:17:58.809+0000 7f2c56df0700 -1 osd.3 128 *** Got signal Terminated *** 2024-05-09T04:17:59.012 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:17:58 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3[39353]: 2024-05-09T04:17:58.809+0000 7f2c56df0700 -1 osd.3 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:17:59.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:17:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:17:59] "GET /metrics HTTP/1.1" 200 38266 "" "Prometheus/2.43.0" 2024-05-09T04:18:00.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:17:59 smithi132 ceph-mon[66347]: osd.3 marked itself down and dead 2024-05-09T04:18:00.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:17:59 smithi078 ceph-mon[83176]: osd.3 marked itself down and dead 2024-05-09T04:18:00.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:17:59 smithi078 ceph-mon[88406]: osd.3 marked itself down and dead 2024-05-09T04:18:00.537 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:00 smithi078 podman[113746]: 2024-05-09 04:18:00.271187654 +0000 UTC m=+1.579019423 container died d2a1e43b69d3d4a39938420fe941572ef4299df32007053746e9ee11c3a23859 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, release=754, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, ceph=True, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-09T04:18:00.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:00 smithi078 ceph-mon[88406]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:18:00.958 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:00 smithi078 ceph-mon[88406]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:18:00.958 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:00 smithi078 ceph-mon[88406]: osdmap e129: 8 total, 7 up, 8 in 2024-05-09T04:18:00.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:00 smithi078 ceph-mon[83176]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:18:00.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:00 smithi078 ceph-mon[83176]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:18:00.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:00 smithi078 ceph-mon[83176]: osdmap e129: 8 total, 7 up, 8 in 2024-05-09T04:18:00.959 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:00 smithi078 podman[113746]: 2024-05-09 04:18:00.636273277 +0000 UTC m=+1.944105044 container cleanup d2a1e43b69d3d4a39938420fe941572ef4299df32007053746e9ee11c3a23859 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, release=754, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, architecture=x86_64, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, distribution-scope=public, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-09T04:18:00.959 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:00 smithi078 bash[113746]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3 2024-05-09T04:18:01.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:00 smithi132 ceph-mon[66347]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:18:01.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:00 smithi132 ceph-mon[66347]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:18:01.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:00 smithi132 ceph-mon[66347]: osdmap e129: 8 total, 7 up, 8 in 2024-05-09T04:18:01.261 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:00 smithi078 podman[113764]: 2024-05-09 04:18:00.946032024 +0000 UTC m=+0.675738714 container remove d2a1e43b69d3d4a39938420fe941572ef4299df32007053746e9ee11c3a23859 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.openshift.expose-services=, vcs-type=git, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, name=centos-stream, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, version=8, io.k8s.display-name=CentOS Stream 8, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc.) 2024-05-09T04:18:01.557 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:01 smithi078 podman[113858]: 2024-05-09 04:18:01.271907006 +0000 UTC m=+0.115119535 container create 7b0c0daf22338713025f44539b86e921d7a53cfc1b43b68d0e105f423af3bee4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240507, GIT_BRANCH=HEAD) 2024-05-09T04:18:01.557 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:01 smithi078 podman[113858]: 2024-05-09 04:18:01.174954312 +0000 UTC m=+0.018166775 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:18:01.558 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:01 smithi078 podman[113858]: 2024-05-09 04:18:01.474584921 +0000 UTC m=+0.317797384 container init 7b0c0daf22338713025f44539b86e921d7a53cfc1b43b68d0e105f423af3bee4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:18:01.558 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:01 smithi078 podman[113858]: 2024-05-09 04:18:01.477912165 +0000 UTC m=+0.321124628 container start 7b0c0daf22338713025f44539b86e921d7a53cfc1b43b68d0e105f423af3bee4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-deactivate, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507) 2024-05-09T04:18:01.558 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:01 smithi078 podman[113858]: 2024-05-09 04:18:01.557753071 +0000 UTC m=+0.400965548 container attach 7b0c0daf22338713025f44539b86e921d7a53cfc1b43b68d0e105f423af3bee4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-05-09T04:18:02.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:01 smithi132 ceph-mon[66347]: osdmap e130: 8 total, 7 up, 8 in 2024-05-09T04:18:02.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:01 smithi078 ceph-mon[83176]: osdmap e130: 8 total, 7 up, 8 in 2024-05-09T04:18:02.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:01 smithi078 ceph-mon[88406]: osdmap e130: 8 total, 7 up, 8 in 2024-05-09T04:18:02.011 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:01 smithi078 podman[113858]: 2024-05-09 04:18:01.774006895 +0000 UTC m=+0.617219351 container died 7b0c0daf22338713025f44539b86e921d7a53cfc1b43b68d0e105f423af3bee4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507) 2024-05-09T04:18:02.725 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:02 smithi078 podman[113884]: 2024-05-09 04:18:02.403353092 +0000 UTC m=+0.624056210 container remove 7b0c0daf22338713025f44539b86e921d7a53cfc1b43b68d0e105f423af3bee4 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:18:02.725 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:02 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.3.service: Deactivated successfully. 2024-05-09T04:18:02.725 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:02 smithi078 systemd[1]: Stopped Ceph osd.3 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:18:02.725 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:02 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.3.service: Consumed 17.598s CPU time. 2024-05-09T04:18:02.726 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:02 smithi078 systemd[1]: Starting Ceph osd.3 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:18:02.997 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:02 smithi078 ceph-mon[88406]: pgmap v149: 161 pgs: 5 active+undersized+degraded, 21 stale+active+clean, 8 active+undersized, 127 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 25/723 objects degraded (3.458%) 2024-05-09T04:18:02.997 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:02 smithi078 ceph-mon[88406]: Health check failed: Degraded data redundancy: 25/723 objects degraded (3.458%), 5 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:02.998 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:02 smithi078 ceph-mon[83176]: pgmap v149: 161 pgs: 5 active+undersized+degraded, 21 stale+active+clean, 8 active+undersized, 127 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 25/723 objects degraded (3.458%) 2024-05-09T04:18:02.998 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:02 smithi078 ceph-mon[83176]: Health check failed: Degraded data redundancy: 25/723 objects degraded (3.458%), 5 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:02.998 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:02 smithi078 podman[113989]: 2024-05-09 04:18:02.765632147 +0000 UTC m=+0.018649975 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:18:02.998 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:02 smithi078 podman[113989]: 2024-05-09 04:18:02.870918041 +0000 UTC m=+0.123935851 container create b53ce33908dce472fe80aa09969ec236a6628a327f43187d47ab4487e465e91f (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate, ceph=True, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-05-09T04:18:03.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:02 smithi132 ceph-mon[66347]: pgmap v149: 161 pgs: 5 active+undersized+degraded, 21 stale+active+clean, 8 active+undersized, 127 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 25/723 objects degraded (3.458%) 2024-05-09T04:18:03.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:02 smithi132 ceph-mon[66347]: Health check failed: Degraded data redundancy: 25/723 objects degraded (3.458%), 5 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:03.261 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 podman[113989]: 2024-05-09 04:18:03.153311983 +0000 UTC m=+0.406329802 container init b53ce33908dce472fe80aa09969ec236a6628a327f43187d47ab4487e465e91f (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504) 2024-05-09T04:18:03.261 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 podman[113989]: 2024-05-09 04:18:03.156954575 +0000 UTC m=+0.409972394 container start b53ce33908dce472fe80aa09969ec236a6628a327f43187d47ab4487e465e91f (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240507, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-05-09T04:18:03.261 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 podman[113989]: 2024-05-09 04:18:03.240458867 +0000 UTC m=+0.493476689 container attach b53ce33908dce472fe80aa09969ec236a6628a327f43187d47ab4487e465e91f (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2) 2024-05-09T04:18:03.986 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate[114005]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:18:03.986 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate[114005]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:18:03.986 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate[114005]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-09T04:18:03.986 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 bash[113989]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:18:03.986 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 bash[113989]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:18:03.986 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 bash[113989]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-09T04:18:03.986 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 bash[113989]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-09T04:18:03.987 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 bash[113989]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-09T04:18:03.987 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 bash[113989]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-09T04:18:03.987 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 bash[113989]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-05-09T04:18:03.987 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 bash[113989]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-09T04:18:03.987 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 bash[113989]: --> ceph-volume raw activate successful for osd ID: 3 2024-05-09T04:18:03.987 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate[114005]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-09T04:18:03.987 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate[114005]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-09T04:18:03.987 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate[114005]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-09T04:18:03.987 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate[114005]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-05-09T04:18:03.987 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate[114005]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-09T04:18:03.987 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate[114005]: --> ceph-volume raw activate successful for osd ID: 3 2024-05-09T04:18:03.987 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:03 smithi078 podman[113989]: 2024-05-09 04:18:03.668421248 +0000 UTC m=+0.921439061 container died b53ce33908dce472fe80aa09969ec236a6628a327f43187d47ab4487e465e91f (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-09T04:18:04.634 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:04 smithi078 podman[114130]: 2024-05-09 04:18:04.30324551 +0000 UTC m=+0.624441183 container remove b53ce33908dce472fe80aa09969ec236a6628a327f43187d47ab4487e465e91f (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-activate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0) 2024-05-09T04:18:04.634 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:04 smithi078 podman[114189]: 2024-05-09 04:18:04.424287568 +0000 UTC m=+0.017951650 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:18:04.634 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:04 smithi078 podman[114189]: 2024-05-09 04:18:04.537352689 +0000 UTC m=+0.131016756 container create cfe832ae10c36d33f5465f876e952857ef7414cb58eb8684c3a0ac8ee5c3073d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.license=GPLv2) 2024-05-09T04:18:05.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:04 smithi132 ceph-mon[66347]: pgmap v150: 161 pgs: 10 active+undersized+degraded, 14 stale+active+clean, 23 active+undersized, 114 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 51/723 objects degraded (7.054%) 2024-05-09T04:18:05.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:04 smithi078 ceph-mon[88406]: pgmap v150: 161 pgs: 10 active+undersized+degraded, 14 stale+active+clean, 23 active+undersized, 114 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 51/723 objects degraded (7.054%) 2024-05-09T04:18:05.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:04 smithi078 ceph-mon[83176]: pgmap v150: 161 pgs: 10 active+undersized+degraded, 14 stale+active+clean, 23 active+undersized, 114 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 51/723 objects degraded (7.054%) 2024-05-09T04:18:05.012 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:04 smithi078 podman[114189]: 2024-05-09 04:18:04.74845918 +0000 UTC m=+0.342123252 container init cfe832ae10c36d33f5465f876e952857ef7414cb58eb8684c3a0ac8ee5c3073d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:18:05.012 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:04 smithi078 podman[114189]: 2024-05-09 04:18:04.751769174 +0000 UTC m=+0.345433242 container start cfe832ae10c36d33f5465f876e952857ef7414cb58eb8684c3a0ac8ee5c3073d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-05-09T04:18:05.012 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:04 smithi078 ceph-osd[114208]: -- 172.21.15.78:0/3235482529 <== mon.1 v2:172.21.15.78:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56236c15d680 con 0x56236c126c00 2024-05-09T04:18:05.012 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:04 smithi078 bash[114189]: cfe832ae10c36d33f5465f876e952857ef7414cb58eb8684c3a0ac8ee5c3073d 2024-05-09T04:18:05.012 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:04 smithi078 systemd[1]: Started Ceph osd.3 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:18:05.719 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:05 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3[114204]: 2024-05-09T04:18:05.290+0000 7fb6a34b87c0 -1 Falling back to public interface 2024-05-09T04:18:05.976 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:05 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:05.976 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:05 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:05.976 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:05 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:05.976 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:05 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:18:05.977 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:05 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:05.977 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:05 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:05.977 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:05 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:05.977 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:05 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:18:06.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:05 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:06.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:05 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:06.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:05 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:06.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:05 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:18:06.996 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:06 smithi078 ceph-mon[88406]: pgmap v151: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-05-09T04:18:06.996 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:06 smithi078 ceph-mon[83176]: pgmap v151: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-05-09T04:18:07.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:06 smithi132 ceph-mon[66347]: pgmap v151: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-05-09T04:18:08.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:07 smithi132 ceph-mon[66347]: Health check update: Degraded data redundancy: 93/723 objects degraded (12.863%), 23 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:08.008 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:07 smithi078 ceph-mon[88406]: Health check update: Degraded data redundancy: 93/723 objects degraded (12.863%), 23 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:08.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:07 smithi078 ceph-mon[83176]: Health check update: Degraded data redundancy: 93/723 objects degraded (12.863%), 23 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:08.261 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:08 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3[114204]: 2024-05-09T04:18:08.007+0000 7fb6a34b87c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-05-09T04:18:08.261 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:08 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3[114204]: 2024-05-09T04:18:08.042+0000 7fb6a34b87c0 -1 osd.3 128 log_to_monitors true 2024-05-09T04:18:08.745 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:08 smithi078 ceph-mon[83176]: pgmap v152: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-05-09T04:18:08.745 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:08 smithi078 ceph-mon[83176]: from='osd.3 [v2:172.21.15.78:6826/3681177972,v1:172.21.15.78:6827/3681177972]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-09T04:18:09.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:08 smithi132 ceph-mon[66347]: pgmap v152: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-05-09T04:18:09.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:08 smithi132 ceph-mon[66347]: from='osd.3 [v2:172.21.15.78:6826/3681177972,v1:172.21.15.78:6827/3681177972]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-09T04:18:09.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:08 smithi078 ceph-mon[88406]: pgmap v152: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-05-09T04:18:09.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:08 smithi078 ceph-mon[88406]: from='osd.3 [v2:172.21.15.78:6826/3681177972,v1:172.21.15.78:6827/3681177972]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-09T04:18:09.011 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:18:08 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3[114204]: 2024-05-09T04:18:08.763+0000 7fb69490f700 -1 osd.3 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T04:18:09.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:18:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:18:09] "GET /metrics HTTP/1.1" 200 38266 "" "Prometheus/2.43.0" 2024-05-09T04:18:10.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:09 smithi132 ceph-mon[66347]: from='osd.3 [v2:172.21.15.78:6826/3681177972,v1:172.21.15.78:6827/3681177972]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-09T04:18:10.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:09 smithi132 ceph-mon[66347]: osdmap e131: 8 total, 7 up, 8 in 2024-05-09T04:18:10.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:09 smithi132 ceph-mon[66347]: from='osd.3 [v2:172.21.15.78:6826/3681177972,v1:172.21.15.78:6827/3681177972]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:18:10.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:09 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:10.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:09 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:10.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:09 smithi078 ceph-mon[83176]: from='osd.3 [v2:172.21.15.78:6826/3681177972,v1:172.21.15.78:6827/3681177972]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-09T04:18:10.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:09 smithi078 ceph-mon[83176]: osdmap e131: 8 total, 7 up, 8 in 2024-05-09T04:18:10.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:09 smithi078 ceph-mon[83176]: from='osd.3 [v2:172.21.15.78:6826/3681177972,v1:172.21.15.78:6827/3681177972]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:18:10.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:09 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:10.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:09 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:10.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:09 smithi078 ceph-mon[88406]: from='osd.3 [v2:172.21.15.78:6826/3681177972,v1:172.21.15.78:6827/3681177972]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-09T04:18:10.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:09 smithi078 ceph-mon[88406]: osdmap e131: 8 total, 7 up, 8 in 2024-05-09T04:18:10.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:09 smithi078 ceph-mon[88406]: from='osd.3 [v2:172.21.15.78:6826/3681177972,v1:172.21.15.78:6827/3681177972]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-09T04:18:10.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:09 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:10.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:09 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:11.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[83176]: pgmap v154: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-05-09T04:18:11.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[83176]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:18:11.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[83176]: osd.3 [v2:172.21.15.78:6826/3681177972,v1:172.21.15.78:6827/3681177972] boot 2024-05-09T04:18:11.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[83176]: osdmap e132: 8 total, 8 up, 8 in 2024-05-09T04:18:11.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:18:11.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:11.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:11.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[88406]: pgmap v154: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-05-09T04:18:11.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[88406]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:18:11.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[88406]: osd.3 [v2:172.21.15.78:6826/3681177972,v1:172.21.15.78:6827/3681177972] boot 2024-05-09T04:18:11.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[88406]: osdmap e132: 8 total, 8 up, 8 in 2024-05-09T04:18:11.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:18:11.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:11.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:10 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:11.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:10 smithi132 ceph-mon[66347]: pgmap v154: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-05-09T04:18:11.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:10 smithi132 ceph-mon[66347]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:18:11.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:10 smithi132 ceph-mon[66347]: osd.3 [v2:172.21.15.78:6826/3681177972,v1:172.21.15.78:6827/3681177972] boot 2024-05-09T04:18:11.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:10 smithi132 ceph-mon[66347]: osdmap e132: 8 total, 8 up, 8 in 2024-05-09T04:18:11.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:10 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-09T04:18:11.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:10 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:11.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:10 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:12.236 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:11 smithi078 ceph-mon[88406]: osdmap e133: 8 total, 8 up, 8 in 2024-05-09T04:18:12.236 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:11 smithi078 ceph-mon[83176]: osdmap e133: 8 total, 8 up, 8 in 2024-05-09T04:18:12.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:11 smithi132 ceph-mon[66347]: osdmap e133: 8 total, 8 up, 8 in 2024-05-09T04:18:12.878 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:12 smithi078 ceph-mon[83176]: pgmap v157: 161 pgs: 7 activating, 20 active+undersized+degraded, 39 active+undersized, 95 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 78/723 objects degraded (10.788%) 2024-05-09T04:18:12.878 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:12 smithi078 ceph-mon[83176]: Health check update: Degraded data redundancy: 78/723 objects degraded (10.788%), 20 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:12.878 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:12 smithi078 ceph-mon[88406]: pgmap v157: 161 pgs: 7 activating, 20 active+undersized+degraded, 39 active+undersized, 95 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 78/723 objects degraded (10.788%) 2024-05-09T04:18:12.878 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:12 smithi078 ceph-mon[88406]: Health check update: Degraded data redundancy: 78/723 objects degraded (10.788%), 20 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:13.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:12 smithi132 ceph-mon[66347]: pgmap v157: 161 pgs: 7 activating, 20 active+undersized+degraded, 39 active+undersized, 95 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 78/723 objects degraded (10.788%) 2024-05-09T04:18:13.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:12 smithi132 ceph-mon[66347]: Health check update: Degraded data redundancy: 78/723 objects degraded (10.788%), 20 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:14.138 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:13 smithi078 ceph-mon[83176]: pgmap v158: 161 pgs: 7 activating, 17 active+undersized+degraded, 29 active+undersized, 108 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 62/723 objects degraded (8.575%) 2024-05-09T04:18:14.139 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:14.139 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:18:14.139 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:13 smithi078 ceph-mon[88406]: pgmap v158: 161 pgs: 7 activating, 17 active+undersized+degraded, 29 active+undersized, 108 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 62/723 objects degraded (8.575%) 2024-05-09T04:18:14.139 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:14.139 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:18:14.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:13 smithi132 ceph-mon[66347]: pgmap v158: 161 pgs: 7 activating, 17 active+undersized+degraded, 29 active+undersized, 108 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 62/723 objects degraded (8.575%) 2024-05-09T04:18:14.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:14.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:18:14.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:18:14 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:18:14.487+0000 7f81013d6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-05-09T04:18:15.627 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:15.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:15.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:15.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:18:15.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:18:15.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:15.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:18:15.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:15.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:15.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:15.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-09T04:18:15.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-09T04:18:15.628 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[83176]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-05-09T04:18:15.629 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:15.629 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:15.629 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:15.629 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:18:15.629 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:18:15.629 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:15.629 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:18:15.629 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:15.629 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:15.629 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:15.629 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-09T04:18:15.630 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-09T04:18:15.630 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:15 smithi078 ceph-mon[88406]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-05-09T04:18:15.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:15.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:15.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:18:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:18:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:18:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-09T04:18:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-09T04:18:15.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:15 smithi132 ceph-mon[66347]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-05-09T04:18:16.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:16 smithi132 ceph-mon[66347]: pgmap v159: 161 pgs: 7 activating, 154 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:16.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:16 smithi132 ceph-mon[66347]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/723 objects degraded (8.575%), 17 pgs degraded) 2024-05-09T04:18:16.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:16 smithi132 ceph-mon[66347]: Cluster is now healthy 2024-05-09T04:18:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:16 smithi078 ceph-mon[83176]: pgmap v159: 161 pgs: 7 activating, 154 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:16 smithi078 ceph-mon[83176]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/723 objects degraded (8.575%), 17 pgs degraded) 2024-05-09T04:18:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:16 smithi078 ceph-mon[83176]: Cluster is now healthy 2024-05-09T04:18:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:16 smithi078 ceph-mon[88406]: pgmap v159: 161 pgs: 7 activating, 154 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:16 smithi078 ceph-mon[88406]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/723 objects degraded (8.575%), 17 pgs degraded) 2024-05-09T04:18:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:16 smithi078 ceph-mon[88406]: Cluster is now healthy 2024-05-09T04:18:19.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:18 smithi132 ceph-mon[66347]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:19.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:18 smithi078 ceph-mon[83176]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:19.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:18 smithi078 ceph-mon[88406]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:19.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:18:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:18:19] "GET /metrics HTTP/1.1" 200 38128 "" "Prometheus/2.43.0" 2024-05-09T04:18:21.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:20 smithi132 ceph-mon[66347]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:21.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:20 smithi078 ceph-mon[83176]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:20 smithi078 ceph-mon[88406]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:23.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:22 smithi132 ceph-mon[66347]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:23.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:22 smithi078 ceph-mon[83176]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:23.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:22 smithi078 ceph-mon[88406]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:24.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:24 smithi132 ceph-mon[66347]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:24.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:24 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:24.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:24 smithi078 ceph-mon[83176]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:24.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:24 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:24.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:24 smithi078 ceph-mon[88406]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:24.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:24 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:25.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:25 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:25.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:25 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:25.334 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:18:25.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:25 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:26.001 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:18:26.001 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (6m) 16s ago 29m 28.0M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:18:26.001 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (6m) 4m ago 28m 80.8M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (8m) 16s ago 28m 67.2M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (8m) 4m ago 33m 463M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (18m) 16s ago 35m 524M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (4m) 16s ago 35m 51.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (5m) 4m ago 33m 36.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (4m) 16s ago 34m 39.8M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (7m) 16s ago 29m 16.6M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (7m) 4m ago 29m 17.3M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (2m) 16s ago 33m 48.0M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (2m) 16s ago 32m 46.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (77s) 16s ago 32m 40.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (21s) 16s ago 32m 12.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a cfe832ae10c3 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (31m) 4m ago 31m 58.2M 3248M 17.2.0 e1d6a67b021e 094794ff5843 2024-05-09T04:18:26.002 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (31m) 4m ago 31m 52.7M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:18:26.003 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (30m) 4m ago 30m 53.5M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:18:26.003 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (30m) 4m ago 30m 57.7M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:18:26.003 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (7m) 4m ago 29m 65.5M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:18:26.003 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (28m) 16s ago 28m 80.2M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:18:26.003 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (28m) 4m ago 28m 80.6M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:18:26.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:26 smithi078 ceph-mon[88406]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-09T04:18:26.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:26 smithi078 ceph-mon[88406]: from='client.54409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:26.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:26 smithi078 ceph-mon[88406]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:26.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:26 smithi078 ceph-mon[83176]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-09T04:18:26.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:26 smithi078 ceph-mon[83176]: from='client.54409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:26.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:26 smithi078 ceph-mon[83176]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:26.412 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:18:26.412 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:18:26.412 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:18:26.412 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:18:26.412 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:18:26.412 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:18:26.412 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:18:26.413 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:18:26.413 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-05-09T04:18:26.413 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 4 2024-05-09T04:18:26.413 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:18:26.413 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:18:26.413 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:18:26.413 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:18:26.413 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:18:26.413 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-05-09T04:18:26.413 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 9 2024-05-09T04:18:26.413 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:18:26.413 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:18:26.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:26 smithi132 ceph-mon[66347]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-09T04:18:26.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:26 smithi132 ceph-mon[66347]: from='client.54409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:26.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:26 smithi132 ceph-mon[66347]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:26.759 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:18:26.759 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:18:26.759 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:18:26.759 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-09T04:18:26.759 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:18:26.759 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "4/8 daemons upgraded", 2024-05-09T04:18:26.759 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-05-09T04:18:26.760 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:18:26.760 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:18:27.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:27 smithi132 ceph-mon[66347]: from='client.44484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:27.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:27 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/4143991511' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:27.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:27 smithi132 ceph-mon[66347]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:27.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:27 smithi078 ceph-mon[83176]: from='client.44484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:27.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:27 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/4143991511' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:27.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:27 smithi078 ceph-mon[83176]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:27.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:27 smithi078 ceph-mon[88406]: from='client.44484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:27.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:27 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/4143991511' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:27.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:27 smithi078 ceph-mon[88406]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:28.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:28 smithi132 ceph-mon[66347]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:18:28.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:28.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:18:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:28 smithi078 ceph-mon[83176]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:18:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:18:28.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:28 smithi078 ceph-mon[88406]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:18:28.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:28.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:18:29.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:18:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:18:29] "GET /metrics HTTP/1.1" 200 38271 "" "Prometheus/2.43.0" 2024-05-09T04:18:30.334 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:30 smithi132 ceph-mon[66347]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:18:30.334 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:30 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-09T04:18:30.334 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:30 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-09T04:18:30.334 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:30 smithi132 ceph-mon[66347]: Upgrade: osd.4 is safe to restart 2024-05-09T04:18:30.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:30 smithi078 ceph-mon[83176]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:18:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:30 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-09T04:18:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:30 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-09T04:18:30.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:30 smithi078 ceph-mon[83176]: Upgrade: osd.4 is safe to restart 2024-05-09T04:18:30.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:30 smithi078 ceph-mon[88406]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:18:30.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:30 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-09T04:18:30.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:30 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-09T04:18:30.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:30 smithi078 ceph-mon[88406]: Upgrade: osd.4 is safe to restart 2024-05-09T04:18:32.408 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:32 smithi132 ceph-mon[66347]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:18:32.408 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:32 smithi132 ceph-mon[66347]: Upgrade: Updating osd.4 2024-05-09T04:18:32.408 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:32.408 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-09T04:18:32.408 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:18:32.408 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:32 smithi132 ceph-mon[66347]: Deploying daemon osd.4 on smithi132 2024-05-09T04:18:32.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:32 smithi078 ceph-mon[83176]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:18:32.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:32 smithi078 ceph-mon[83176]: Upgrade: Updating osd.4 2024-05-09T04:18:32.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:32.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-09T04:18:32.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:18:32.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:32 smithi078 ceph-mon[83176]: Deploying daemon osd.4 on smithi132 2024-05-09T04:18:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:32 smithi078 ceph-mon[88406]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:18:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:32 smithi078 ceph-mon[88406]: Upgrade: Updating osd.4 2024-05-09T04:18:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-09T04:18:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:18:32.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:32 smithi078 ceph-mon[88406]: Deploying daemon osd.4 on smithi132 2024-05-09T04:18:34.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:34 smithi132 ceph-mon[66347]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:18:34.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:34 smithi078 ceph-mon[83176]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:18:34.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:34 smithi078 ceph-mon[88406]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:18:34.605 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:34 smithi132 systemd[1]: Stopping Ceph osd.4 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:18:35.003 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[27170]: 2024-05-09T04:18:34.604+0000 7fd4843e3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:18:35.003 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[27170]: 2024-05-09T04:18:34.604+0000 7fd4843e3700 -1 osd.4 133 *** Got signal Terminated *** 2024-05-09T04:18:35.003 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[27170]: 2024-05-09T04:18:34.604+0000 7fd4843e3700 -1 osd.4 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:18:35.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:35 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:35.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:35 smithi132 ceph-mon[66347]: osd.4 marked itself down and dead 2024-05-09T04:18:35.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:35 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:35.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:35 smithi078 ceph-mon[83176]: osd.4 marked itself down and dead 2024-05-09T04:18:35.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:35 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:35.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:35 smithi078 ceph-mon[88406]: osd.4 marked itself down and dead 2024-05-09T04:18:35.931 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:35 smithi132 podman[76452]: 2024-05-09 04:18:35.63850018 +0000 UTC m=+1.136765252 container died 094794ff5843e037b2130e477da7425338d497cf71d9cce9e887a7aae8fe2d9e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4, io.openshift.expose-services=, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vcs-type=git, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, version=8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0) 2024-05-09T04:18:36.192 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:36 smithi132 podman[76452]: 2024-05-09 04:18:36.055278187 +0000 UTC m=+1.553543240 container cleanup 094794ff5843e037b2130e477da7425338d497cf71d9cce9e887a7aae8fe2d9e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, version=8, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, release=754, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., vcs-type=git, RELEASE=HEAD, io.buildah.version=1.19.8) 2024-05-09T04:18:36.192 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:36 smithi132 bash[76452]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4 2024-05-09T04:18:36.503 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:36 smithi132 podman[76472]: 2024-05-09 04:18:36.240332098 +0000 UTC m=+0.605602557 container remove 094794ff5843e037b2130e477da7425338d497cf71d9cce9e887a7aae8fe2d9e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, vendor=Red Hat, Inc., RELEASE=HEAD, io.buildah.version=1.19.8, vcs-type=git, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=) 2024-05-09T04:18:36.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:36 smithi132 ceph-mon[66347]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:18:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:36 smithi132 ceph-mon[66347]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:18:36.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:36 smithi132 ceph-mon[66347]: osdmap e134: 8 total, 7 up, 8 in 2024-05-09T04:18:36.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:36 smithi078 ceph-mon[88406]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:18:36.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:36 smithi078 ceph-mon[88406]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:18:36.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:36 smithi078 ceph-mon[88406]: osdmap e134: 8 total, 7 up, 8 in 2024-05-09T04:18:36.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:36 smithi078 ceph-mon[83176]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:18:36.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:36 smithi078 ceph-mon[83176]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:18:36.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:36 smithi078 ceph-mon[83176]: osdmap e134: 8 total, 7 up, 8 in 2024-05-09T04:18:36.994 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:36 smithi132 podman[76568]: 2024-05-09 04:18:36.579844695 +0000 UTC m=+0.018228331 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:18:36.994 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:36 smithi132 podman[76568]: 2024-05-09 04:18:36.691739851 +0000 UTC m=+0.130123472 container create 2b98b56cf20270d2c55ade495e2e274a810a30df55d5ecf9ad5ca780cb9e5a7e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-deactivate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-05-09T04:18:36.994 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:36 smithi132 podman[76568]: 2024-05-09 04:18:36.902808885 +0000 UTC m=+0.341192514 container init 2b98b56cf20270d2c55ade495e2e274a810a30df55d5ecf9ad5ca780cb9e5a7e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240507, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-05-09T04:18:36.994 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:36 smithi132 podman[76568]: 2024-05-09 04:18:36.906253016 +0000 UTC m=+0.344636642 container start 2b98b56cf20270d2c55ade495e2e274a810a30df55d5ecf9ad5ca780cb9e5a7e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux ) 2024-05-09T04:18:36.994 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:36 smithi132 podman[76568]: 2024-05-09 04:18:36.994420178 +0000 UTC m=+0.432803809 container attach 2b98b56cf20270d2c55ade495e2e274a810a30df55d5ecf9ad5ca780cb9e5a7e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504) 2024-05-09T04:18:37.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:37 smithi132 ceph-mon[66347]: osdmap e135: 8 total, 7 up, 8 in 2024-05-09T04:18:37.253 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:37 smithi132 podman[76568]: 2024-05-09 04:18:37.201683262 +0000 UTC m=+0.640066881 container died 2b98b56cf20270d2c55ade495e2e274a810a30df55d5ecf9ad5ca780cb9e5a7e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-deactivate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:18:37.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:37 smithi078 ceph-mon[83176]: osdmap e135: 8 total, 7 up, 8 in 2024-05-09T04:18:37.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:37 smithi078 ceph-mon[88406]: osdmap e135: 8 total, 7 up, 8 in 2024-05-09T04:18:38.199 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:37 smithi132 podman[76595]: 2024-05-09 04:18:37.823711508 +0000 UTC m=+0.616652518 container remove 2b98b56cf20270d2c55ade495e2e274a810a30df55d5ecf9ad5ca780cb9e5a7e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.build-date=20240507, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:18:38.199 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:37 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.4.service: Deactivated successfully. 2024-05-09T04:18:38.199 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:37 smithi132 systemd[1]: Stopped Ceph osd.4 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:18:38.199 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:37 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.4.service: Consumed 17.573s CPU time. 2024-05-09T04:18:38.200 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:38 smithi132 systemd[1]: Starting Ceph osd.4 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:18:38.503 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:38 smithi132 podman[76703]: 2024-05-09 04:18:38.171940857 +0000 UTC m=+0.018415467 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:18:38.503 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:38 smithi132 podman[76703]: 2024-05-09 04:18:38.300374696 +0000 UTC m=+0.146849290 container create af17ef95ffdae173a55c61f9c26bbbb2806522fb26d59a3cbd11005e0759845d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-05-09T04:18:38.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:38 smithi132 ceph-mon[66347]: pgmap v172: 161 pgs: 20 peering, 18 stale+active+clean, 123 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:38.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:38 smithi132 ceph-mon[66347]: Health check failed: Reduced data availability: 2 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-05-09T04:18:38.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:38 smithi078 ceph-mon[83176]: pgmap v172: 161 pgs: 20 peering, 18 stale+active+clean, 123 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:38.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:38 smithi078 ceph-mon[83176]: Health check failed: Reduced data availability: 2 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-05-09T04:18:38.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:38 smithi078 ceph-mon[88406]: pgmap v172: 161 pgs: 20 peering, 18 stale+active+clean, 123 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:18:38.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:38 smithi078 ceph-mon[88406]: Health check failed: Reduced data availability: 2 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-05-09T04:18:39.001 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:38 smithi132 podman[76703]: 2024-05-09 04:18:38.511803498 +0000 UTC m=+0.358278096 container init af17ef95ffdae173a55c61f9c26bbbb2806522fb26d59a3cbd11005e0759845d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:18:39.001 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:38 smithi132 podman[76703]: 2024-05-09 04:18:38.515487531 +0000 UTC m=+0.361962126 container start af17ef95ffdae173a55c61f9c26bbbb2806522fb26d59a3cbd11005e0759845d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507) 2024-05-09T04:18:39.001 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:38 smithi132 podman[76703]: 2024-05-09 04:18:38.628249863 +0000 UTC m=+0.474724458 container attach af17ef95ffdae173a55c61f9c26bbbb2806522fb26d59a3cbd11005e0759845d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:18:39.251 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate[76718]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:18:39.252 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate[76718]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:18:39.252 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 bash[76703]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:18:39.252 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 bash[76703]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:18:39.252 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 bash[76703]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-09T04:18:39.252 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 bash[76703]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-09T04:18:39.252 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 bash[76703]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-09T04:18:39.252 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 bash[76703]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-09T04:18:39.252 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 bash[76703]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-05-09T04:18:39.252 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 bash[76703]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-09T04:18:39.252 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 bash[76703]: --> ceph-volume raw activate successful for osd ID: 4 2024-05-09T04:18:39.252 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate[76718]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-09T04:18:39.252 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate[76718]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-09T04:18:39.253 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate[76718]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-09T04:18:39.253 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate[76718]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-09T04:18:39.253 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate[76718]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-05-09T04:18:39.253 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate[76718]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-09T04:18:39.253 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate[76718]: --> ceph-volume raw activate successful for osd ID: 4 2024-05-09T04:18:39.253 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 podman[76703]: 2024-05-09 04:18:39.025909375 +0000 UTC m=+0.872383973 container died af17ef95ffdae173a55c61f9c26bbbb2806522fb26d59a3cbd11005e0759845d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-05-09T04:18:39.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:39 smithi132 ceph-mon[66347]: Health check failed: Degraded data redundancy: 77/723 objects degraded (10.650%), 16 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:39.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:39 smithi078 ceph-mon[83176]: Health check failed: Degraded data redundancy: 77/723 objects degraded (10.650%), 16 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:39.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:39 smithi078 ceph-mon[88406]: Health check failed: Degraded data redundancy: 77/723 objects degraded (10.650%), 16 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:39.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:18:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:18:39] "GET /metrics HTTP/1.1" 200 38271 "" "Prometheus/2.43.0" 2024-05-09T04:18:39.833 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 podman[76855]: 2024-05-09 04:18:39.583012376 +0000 UTC m=+0.551482372 container remove af17ef95ffdae173a55c61f9c26bbbb2806522fb26d59a3cbd11005e0759845d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-05-09T04:18:39.833 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 podman[76925]: 2024-05-09 04:18:39.70389493 +0000 UTC m=+0.018107221 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:18:40.102 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:39 smithi132 podman[76925]: 2024-05-09 04:18:39.834125076 +0000 UTC m=+0.148337351 container create cdf9ea05dcb05084f432ba81f7a0901a5577e8672bb7bfaeda80684901b67f7c (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-05-09T04:18:40.102 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:40 smithi132 podman[76925]: 2024-05-09 04:18:40.078852854 +0000 UTC m=+0.393065136 container init cdf9ea05dcb05084f432ba81f7a0901a5577e8672bb7bfaeda80684901b67f7c (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-05-09T04:18:40.102 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:40 smithi132 podman[76925]: 2024-05-09 04:18:40.08232629 +0000 UTC m=+0.396538562 container start cdf9ea05dcb05084f432ba81f7a0901a5577e8672bb7bfaeda80684901b67f7c (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:18:40.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:40 smithi132 ceph-mon[66347]: pgmap v173: 161 pgs: 16 active+undersized+degraded, 20 peering, 32 active+undersized, 93 active+clean; 585 KiB data, 171 MiB used, 715 GiB / 715 GiB avail; 77/723 objects degraded (10.650%) 2024-05-09T04:18:40.503 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:40 smithi132 ceph-osd[76945]: -- 172.21.15.132:0/1655264148 <== mon.2 v2:172.21.15.132:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55baa42d7680 con 0x55baa42a1400 2024-05-09T04:18:40.504 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:40 smithi132 bash[76925]: cdf9ea05dcb05084f432ba81f7a0901a5577e8672bb7bfaeda80684901b67f7c 2024-05-09T04:18:40.504 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:40 smithi132 systemd[1]: Started Ceph osd.4 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:18:40.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:40 smithi078 ceph-mon[83176]: pgmap v173: 161 pgs: 16 active+undersized+degraded, 20 peering, 32 active+undersized, 93 active+clean; 585 KiB data, 171 MiB used, 715 GiB / 715 GiB avail; 77/723 objects degraded (10.650%) 2024-05-09T04:18:40.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:40 smithi078 ceph-mon[88406]: pgmap v173: 161 pgs: 16 active+undersized+degraded, 20 peering, 32 active+undersized, 93 active+clean; 585 KiB data, 171 MiB used, 715 GiB / 715 GiB avail; 77/723 objects degraded (10.650%) 2024-05-09T04:18:41.003 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:40 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:18:40.629+0000 7f3b411807c0 -1 Falling back to public interface 2024-05-09T04:18:41.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:41 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:41.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:41 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:41.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:41 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:18:41.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:41 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:41.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:41 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:41.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:41 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:18:41.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:41 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:41.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:41 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:41.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:41 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:18:42.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:42 smithi132 ceph-mon[66347]: pgmap v174: 161 pgs: 19 active+undersized+degraded, 9 peering, 40 active+undersized, 93 active+clean; 585 KiB data, 171 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-05-09T04:18:42.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:42 smithi078 ceph-mon[83176]: pgmap v174: 161 pgs: 19 active+undersized+degraded, 9 peering, 40 active+undersized, 93 active+clean; 585 KiB data, 171 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-05-09T04:18:42.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:42 smithi078 ceph-mon[88406]: pgmap v174: 161 pgs: 19 active+undersized+degraded, 9 peering, 40 active+undersized, 93 active+clean; 585 KiB data, 171 MiB used, 715 GiB / 715 GiB avail; 95/723 objects degraded (13.140%) 2024-05-09T04:18:43.408 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:18:43.111+0000 7f3b411807c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-05-09T04:18:43.408 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:18:43.148+0000 7f3b411807c0 -1 osd.4 133 log_to_monitors true 2024-05-09T04:18:43.743 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:43 smithi132 ceph-mon[66347]: Health check update: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-05-09T04:18:43.743 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:43.743 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:18:43.743 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:43 smithi132 ceph-mon[66347]: from='osd.4 [v2:172.21.15.132:6800/1241653389,v1:172.21.15.132:6801/1241653389]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-09T04:18:43.743 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:43 smithi132 ceph-mon[66347]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-09T04:18:43.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:43 smithi078 ceph-mon[83176]: Health check update: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-05-09T04:18:43.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:43.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:18:43.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:43 smithi078 ceph-mon[83176]: from='osd.4 [v2:172.21.15.132:6800/1241653389,v1:172.21.15.132:6801/1241653389]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-09T04:18:43.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:43 smithi078 ceph-mon[83176]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-09T04:18:43.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:43 smithi078 ceph-mon[88406]: Health check update: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-05-09T04:18:43.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:43.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:18:43.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:43 smithi078 ceph-mon[88406]: from='osd.4 [v2:172.21.15.132:6800/1241653389,v1:172.21.15.132:6801/1241653389]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-09T04:18:43.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:43 smithi078 ceph-mon[88406]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-09T04:18:44.315 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:18:44 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:18:44.007+0000 7f3b325d7700 -1 osd.4 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T04:18:44.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:44 smithi132 ceph-mon[66347]: pgmap v175: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 171 MiB used, 715 GiB / 715 GiB avail; 123/723 objects degraded (17.012%) 2024-05-09T04:18:44.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:44 smithi132 ceph-mon[66347]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-05-09T04:18:44.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:44 smithi132 ceph-mon[66347]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-09T04:18:44.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:44 smithi132 ceph-mon[66347]: osdmap e136: 8 total, 7 up, 8 in 2024-05-09T04:18:44.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:44 smithi132 ceph-mon[66347]: from='osd.4 [v2:172.21.15.132:6800/1241653389,v1:172.21.15.132:6801/1241653389]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:18:44.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:44 smithi132 ceph-mon[66347]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:18:44.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:44 smithi078 ceph-mon[88406]: pgmap v175: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 171 MiB used, 715 GiB / 715 GiB avail; 123/723 objects degraded (17.012%) 2024-05-09T04:18:44.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:44 smithi078 ceph-mon[88406]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-05-09T04:18:44.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:44 smithi078 ceph-mon[88406]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-09T04:18:44.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:44 smithi078 ceph-mon[88406]: osdmap e136: 8 total, 7 up, 8 in 2024-05-09T04:18:44.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:44 smithi078 ceph-mon[88406]: from='osd.4 [v2:172.21.15.132:6800/1241653389,v1:172.21.15.132:6801/1241653389]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:18:44.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:44 smithi078 ceph-mon[88406]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:18:44.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:44 smithi078 ceph-mon[83176]: pgmap v175: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 171 MiB used, 715 GiB / 715 GiB avail; 123/723 objects degraded (17.012%) 2024-05-09T04:18:44.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:44 smithi078 ceph-mon[83176]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-05-09T04:18:44.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:44 smithi078 ceph-mon[83176]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-09T04:18:44.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:44 smithi078 ceph-mon[83176]: osdmap e136: 8 total, 7 up, 8 in 2024-05-09T04:18:44.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:44 smithi078 ceph-mon[83176]: from='osd.4 [v2:172.21.15.132:6800/1241653389,v1:172.21.15.132:6801/1241653389]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:18:44.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:44 smithi078 ceph-mon[83176]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:18:45.668 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:45.668 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:45.669 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:45.669 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[83176]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:18:45.669 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[83176]: osd.4 [v2:172.21.15.132:6800/1241653389,v1:172.21.15.132:6801/1241653389] boot 2024-05-09T04:18:45.669 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[83176]: osdmap e137: 8 total, 8 up, 8 in 2024-05-09T04:18:45.669 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:18:45.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:45.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:45.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:45.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[88406]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:18:45.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[88406]: osd.4 [v2:172.21.15.132:6800/1241653389,v1:172.21.15.132:6801/1241653389] boot 2024-05-09T04:18:45.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[88406]: osdmap e137: 8 total, 8 up, 8 in 2024-05-09T04:18:45.670 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:45 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:18:45.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:45 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:45.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:45 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:45.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:45 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:45.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:45 smithi132 ceph-mon[66347]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:18:45.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:45 smithi132 ceph-mon[66347]: osd.4 [v2:172.21.15.132:6800/1241653389,v1:172.21.15.132:6801/1241653389] boot 2024-05-09T04:18:45.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:45 smithi132 ceph-mon[66347]: osdmap e137: 8 total, 8 up, 8 in 2024-05-09T04:18:45.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:45 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-09T04:18:46.749 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:46 smithi132 ceph-mon[66347]: pgmap v177: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 181 MiB used, 715 GiB / 715 GiB avail; 233 B/s rd, 0 op/s; 123/723 objects degraded (17.012%) 2024-05-09T04:18:46.749 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:46 smithi132 ceph-mon[66347]: osdmap e138: 8 total, 8 up, 8 in 2024-05-09T04:18:46.749 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:46 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:46.749 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:46 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:46.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:46 smithi078 ceph-mon[83176]: pgmap v177: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 181 MiB used, 715 GiB / 715 GiB avail; 233 B/s rd, 0 op/s; 123/723 objects degraded (17.012%) 2024-05-09T04:18:46.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:46 smithi078 ceph-mon[83176]: osdmap e138: 8 total, 8 up, 8 in 2024-05-09T04:18:46.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:46 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:46.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:46 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:46.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:46 smithi078 ceph-mon[88406]: pgmap v177: 161 pgs: 24 active+undersized+degraded, 44 active+undersized, 93 active+clean; 585 KiB data, 181 MiB used, 715 GiB / 715 GiB avail; 233 B/s rd, 0 op/s; 123/723 objects degraded (17.012%) 2024-05-09T04:18:46.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:46 smithi078 ceph-mon[88406]: osdmap e138: 8 total, 8 up, 8 in 2024-05-09T04:18:46.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:46 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:46.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:46 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:47.573 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:47 smithi132 ceph-mon[66347]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 20 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:47.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:47 smithi078 ceph-mon[83176]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 20 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:47.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:47 smithi078 ceph-mon[88406]: Health check update: Degraded data redundancy: 95/723 objects degraded (13.140%), 20 pgs degraded (PG_DEGRADED) 2024-05-09T04:18:48.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:48 smithi132 ceph-mon[66347]: pgmap v180: 161 pgs: 8 peering, 20 active+undersized+degraded, 33 active+undersized, 100 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 95/723 objects degraded (13.140%) 2024-05-09T04:18:48.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:48 smithi078 ceph-mon[88406]: pgmap v180: 161 pgs: 8 peering, 20 active+undersized+degraded, 33 active+undersized, 100 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 95/723 objects degraded (13.140%) 2024-05-09T04:18:48.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:48 smithi078 ceph-mon[83176]: pgmap v180: 161 pgs: 8 peering, 20 active+undersized+degraded, 33 active+undersized, 100 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 95/723 objects degraded (13.140%) 2024-05-09T04:18:49.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:18:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:18:49] "GET /metrics HTTP/1.1" 200 38341 "" "Prometheus/2.43.0" 2024-05-09T04:18:50.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:18:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:18:49.744+0000 7f81013d6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-09T04:18:51.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: pgmap v181: 161 pgs: 8 peering, 10 active+undersized+degraded, 13 active+undersized, 130 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 36/723 objects degraded (4.979%) 2024-05-09T04:18:51.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:51.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:51.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:18:51.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:18:51.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:51.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:18:51.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:51.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:51.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:51.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-09T04:18:51.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-09T04:18:51.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:50 smithi132 ceph-mon[66347]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-09T04:18:51.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: pgmap v181: 161 pgs: 8 peering, 10 active+undersized+degraded, 13 active+undersized, 130 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 36/723 objects degraded (4.979%) 2024-05-09T04:18:51.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:51.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:51.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:18:51.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:18:51.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:51.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:18:51.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:51.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:51.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[83176]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: pgmap v181: 161 pgs: 8 peering, 10 active+undersized+degraded, 13 active+undersized, 130 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 36/723 objects degraded (4.979%) 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:51.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:51.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-09T04:18:51.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-09T04:18:51.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:50 smithi078 ceph-mon[88406]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-09T04:18:52.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:51 smithi132 ceph-mon[66347]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/723 objects degraded (4.979%), 10 pgs degraded) 2024-05-09T04:18:52.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:51 smithi132 ceph-mon[66347]: Cluster is now healthy 2024-05-09T04:18:52.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:51 smithi078 ceph-mon[83176]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/723 objects degraded (4.979%), 10 pgs degraded) 2024-05-09T04:18:52.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:51 smithi078 ceph-mon[83176]: Cluster is now healthy 2024-05-09T04:18:52.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:51 smithi078 ceph-mon[88406]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/723 objects degraded (4.979%), 10 pgs degraded) 2024-05-09T04:18:52.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:51 smithi078 ceph-mon[88406]: Cluster is now healthy 2024-05-09T04:18:53.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:52 smithi132 ceph-mon[66347]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-05-09T04:18:53.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:52 smithi078 ceph-mon[83176]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-05-09T04:18:53.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:52 smithi078 ceph-mon[88406]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-05-09T04:18:55.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:54 smithi132 ceph-mon[66347]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-09T04:18:55.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:54 smithi078 ceph-mon[83176]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-09T04:18:55.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:54 smithi078 ceph-mon[88406]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-09T04:18:56.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:55 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:56.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:55 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:56.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:55 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:18:57.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:56 smithi132 ceph-mon[66347]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-05-09T04:18:57.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:56 smithi078 ceph-mon[83176]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-05-09T04:18:57.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:56 smithi078 ceph-mon[88406]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-05-09T04:18:57.126 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:18:57.798 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:18:57.798 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (7m) 48s ago 29m 28.0M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:18:57.798 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (7m) 12s ago 29m 80.0M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:18:57.799 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (8m) 48s ago 28m 67.2M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:18:57.799 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (8m) 12s ago 34m 468M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:18:57.799 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (18m) 48s ago 35m 524M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:18:57.799 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (5m) 48s ago 35m 51.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:18:57.799 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (6m) 12s ago 34m 45.5M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:18:57.799 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (4m) 48s ago 34m 39.8M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:18:57.799 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (8m) 48s ago 30m 16.6M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:18:57.799 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (8m) 12s ago 30m 17.3M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:18:57.799 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (3m) 48s ago 33m 48.0M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:18:57.799 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (2m) 48s ago 33m 46.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:18:57.799 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (109s) 48s ago 33m 40.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:18:57.800 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (53s) 48s ago 32m 12.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a cfe832ae10c3 2024-05-09T04:18:57.800 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (18s) 12s ago 32m 12.0M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a cdf9ea05dcb0 2024-05-09T04:18:57.800 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (31m) 12s ago 31m 56.7M 3248M 17.2.0 e1d6a67b021e 2fc16641cca7 2024-05-09T04:18:57.800 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (31m) 12s ago 31m 56.4M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:18:57.800 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (31m) 12s ago 31m 62.0M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:18:57.800 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (7m) 12s ago 30m 70.0M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:18:57.800 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (28m) 48s ago 28m 80.2M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:18:57.800 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (28m) 12s ago 28m 82.0M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:18:58.212 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:18:58.212 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:18:58.212 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:18:58.212 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:18:58.212 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:18:58.212 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:18:58.212 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:18:58.212 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:18:58.212 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-05-09T04:18:58.212 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 5 2024-05-09T04:18:58.213 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:18:58.213 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:18:58.213 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:18:58.213 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:18:58.213 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:18:58.213 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-05-09T04:18:58.213 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 10 2024-05-09T04:18:58.213 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:18:58.213 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:18:58.551 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:18:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:18:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:18:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-09T04:18:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:18:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "5/8 daemons upgraded", 2024-05-09T04:18:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-05-09T04:18:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:18:58.552 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:18:59.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:58 smithi132 ceph-mon[66347]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 0 op/s 2024-05-09T04:18:59.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:58 smithi132 ceph-mon[66347]: from='client.44505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:59.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:58 smithi132 ceph-mon[66347]: from='client.54451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:59.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:58 smithi132 ceph-mon[66347]: from='client.54457 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:59.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:59.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:18:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:59.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:58 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2086303586' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:59.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[88406]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 0 op/s 2024-05-09T04:18:59.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[88406]: from='client.44505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:59.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[88406]: from='client.54451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:59.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[88406]: from='client.54457 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:59.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:59.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:18:59.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:59.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/2086303586' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:59.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[83176]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 0 op/s 2024-05-09T04:18:59.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[83176]: from='client.44505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:59.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[83176]: from='client.54451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:59.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[83176]: from='client.54457 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:18:59.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:59.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:18:59.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:18:59.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:58 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/2086303586' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:18:59.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:18:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:18:59] "GET /metrics HTTP/1.1" 200 38342 "" "Prometheus/2.43.0" 2024-05-09T04:19:00.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:18:59 smithi132 ceph-mon[66347]: from='client.54469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:00.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:18:59 smithi078 ceph-mon[83176]: from='client.54469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:00.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:18:59 smithi078 ceph-mon[88406]: from='client.54469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:01.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:00 smithi132 ceph-mon[66347]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:01.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:00 smithi078 ceph-mon[83176]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:01.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:00 smithi078 ceph-mon[88406]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:03.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:02 smithi132 ceph-mon[66347]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:03.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:02 smithi078 ceph-mon[83176]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:03.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:02 smithi078 ceph-mon[88406]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:05.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:04 smithi132 ceph-mon[66347]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:05.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:04 smithi078 ceph-mon[88406]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:05.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:04 smithi078 ceph-mon[83176]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:06.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:05 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:06.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:05 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-09T04:19:06.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:05 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-09T04:19:06.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:05 smithi132 ceph-mon[66347]: Upgrade: osd.5 is safe to restart 2024-05-09T04:19:06.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:05 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:06.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:05 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-09T04:19:06.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:05 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-09T04:19:06.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:05 smithi078 ceph-mon[88406]: Upgrade: osd.5 is safe to restart 2024-05-09T04:19:06.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:05 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:06.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:05 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-09T04:19:06.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:05 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-09T04:19:06.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:05 smithi078 ceph-mon[83176]: Upgrade: osd.5 is safe to restart 2024-05-09T04:19:07.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:06 smithi132 ceph-mon[66347]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:07.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:06 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:07.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:06 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-09T04:19:07.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:06 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:19:07.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:06 smithi078 ceph-mon[88406]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:07.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:06 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:07.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:06 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-09T04:19:07.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:06 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:19:07.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:06 smithi078 ceph-mon[83176]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:07.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:06 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:07.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:06 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-09T04:19:07.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:06 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:19:07.957 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:07 smithi132 ceph-mon[66347]: Upgrade: Updating osd.5 2024-05-09T04:19:07.957 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:07 smithi132 ceph-mon[66347]: Deploying daemon osd.5 on smithi132 2024-05-09T04:19:08.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:07 smithi078 ceph-mon[88406]: Upgrade: Updating osd.5 2024-05-09T04:19:08.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:07 smithi078 ceph-mon[88406]: Deploying daemon osd.5 on smithi132 2024-05-09T04:19:08.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:07 smithi078 ceph-mon[83176]: Upgrade: Updating osd.5 2024-05-09T04:19:08.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:07 smithi078 ceph-mon[83176]: Deploying daemon osd.5 on smithi132 2024-05-09T04:19:09.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:08 smithi132 ceph-mon[66347]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:09.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:08 smithi078 ceph-mon[83176]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:09.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:08 smithi078 ceph-mon[88406]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:09.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:19:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:19:09] "GET /metrics HTTP/1.1" 200 38342 "" "Prometheus/2.43.0" 2024-05-09T04:19:09.929 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:09 smithi132 systemd[1]: Stopping Ceph osd.5 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:19:10.253 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:09 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[30119]: 2024-05-09T04:19:09.928+0000 7f4ad6952700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:19:10.253 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:09 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[30119]: 2024-05-09T04:19:09.928+0000 7f4ad6952700 -1 osd.5 138 *** Got signal Terminated *** 2024-05-09T04:19:10.253 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:09 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[30119]: 2024-05-09T04:19:09.928+0000 7f4ad6952700 -1 osd.5 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:19:11.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:10 smithi132 ceph-mon[66347]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:11.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:10 smithi132 ceph-mon[66347]: osd.5 marked itself down and dead 2024-05-09T04:19:11.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:10 smithi078 ceph-mon[83176]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:11.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:10 smithi078 ceph-mon[83176]: osd.5 marked itself down and dead 2024-05-09T04:19:11.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:10 smithi078 ceph-mon[88406]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:11.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:10 smithi078 ceph-mon[88406]: osd.5 marked itself down and dead 2024-05-09T04:19:11.522 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:11 smithi132 podman[80992]: 2024-05-09 04:19:11.257242356 +0000 UTC m=+1.457876518 container died 2fc16641cca7c0fc291e9f6216b31ae7ae142a5e71499bd2eb3d36c8c4f92f0d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, version=8, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-09T04:19:11.846 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:11 smithi132 ceph-mon[66347]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:19:11.846 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:11 smithi132 ceph-mon[66347]: osdmap e139: 8 total, 7 up, 8 in 2024-05-09T04:19:11.847 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:11 smithi132 podman[80992]: 2024-05-09 04:19:11.603521095 +0000 UTC m=+1.804155250 container cleanup 2fc16641cca7c0fc291e9f6216b31ae7ae142a5e71499bd2eb3d36c8c4f92f0d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, ceph=True, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , vcs-type=git, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, version=8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, name=centos-stream, io.buildah.version=1.19.8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD) 2024-05-09T04:19:11.847 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:11 smithi132 bash[80992]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5 2024-05-09T04:19:12.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:11 smithi078 ceph-mon[83176]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:19:12.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:11 smithi078 ceph-mon[83176]: osdmap e139: 8 total, 7 up, 8 in 2024-05-09T04:19:12.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:11 smithi078 ceph-mon[88406]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:19:12.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:11 smithi078 ceph-mon[88406]: osdmap e139: 8 total, 7 up, 8 in 2024-05-09T04:19:12.253 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:11 smithi132 podman[81009]: 2024-05-09 04:19:11.846786355 +0000 UTC m=+0.585207112 container remove 2fc16641cca7c0fc291e9f6216b31ae7ae142a5e71499bd2eb3d36c8c4f92f0d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.openshift.expose-services=, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, ceph=True, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, release=754, version=8) 2024-05-09T04:19:12.541 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:12 smithi132 podman[81103]: 2024-05-09 04:19:12.180042502 +0000 UTC m=+0.018885814 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:19:12.541 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:12 smithi132 podman[81103]: 2024-05-09 04:19:12.297722793 +0000 UTC m=+0.136566076 container create 6775460706a6a451a57099ca107b79db8264e71a758572897ab76990d3a75f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-05-09T04:19:12.541 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:12 smithi132 podman[81103]: 2024-05-09 04:19:12.542365831 +0000 UTC m=+0.381209116 container init 6775460706a6a451a57099ca107b79db8264e71a758572897ab76990d3a75f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2) 2024-05-09T04:19:12.841 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:12 smithi132 podman[81103]: 2024-05-09 04:19:12.545709746 +0000 UTC m=+0.384553034 container start 6775460706a6a451a57099ca107b79db8264e71a758572897ab76990d3a75f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-05-09T04:19:12.841 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:12 smithi132 podman[81103]: 2024-05-09 04:19:12.650624709 +0000 UTC m=+0.489467996 container attach 6775460706a6a451a57099ca107b79db8264e71a758572897ab76990d3a75f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-deactivate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-05-09T04:19:12.841 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:12 smithi132 ceph-mon[66347]: pgmap v193: 161 pgs: 12 peering, 17 stale+active+clean, 132 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:19:12.842 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:12 smithi132 ceph-mon[66347]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-05-09T04:19:12.842 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:12 smithi132 ceph-mon[66347]: osdmap e140: 8 total, 7 up, 8 in 2024-05-09T04:19:13.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:12 smithi078 ceph-mon[83176]: pgmap v193: 161 pgs: 12 peering, 17 stale+active+clean, 132 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:19:13.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:12 smithi078 ceph-mon[83176]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-05-09T04:19:13.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:12 smithi078 ceph-mon[83176]: osdmap e140: 8 total, 7 up, 8 in 2024-05-09T04:19:13.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:12 smithi078 ceph-mon[88406]: pgmap v193: 161 pgs: 12 peering, 17 stale+active+clean, 132 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:19:13.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:12 smithi078 ceph-mon[88406]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-05-09T04:19:13.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:12 smithi078 ceph-mon[88406]: osdmap e140: 8 total, 7 up, 8 in 2024-05-09T04:19:13.187 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:12 smithi132 podman[81103]: 2024-05-09 04:19:12.842381724 +0000 UTC m=+0.681225020 container died 6775460706a6a451a57099ca107b79db8264e71a758572897ab76990d3a75f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.build-date=20240507) 2024-05-09T04:19:13.753 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:13 smithi132 podman[81129]: 2024-05-09 04:19:13.473373596 +0000 UTC m=+0.625271145 container remove 6775460706a6a451a57099ca107b79db8264e71a758572897ab76990d3a75f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-05-09T04:19:13.753 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:13 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.5.service: Deactivated successfully. 2024-05-09T04:19:13.753 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:13 smithi132 systemd[1]: Stopped Ceph osd.5 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:19:13.754 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:13 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.5.service: Consumed 16.872s CPU time. 2024-05-09T04:19:13.754 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:13 smithi132 systemd[1]: Starting Ceph osd.5 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:19:14.177 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:14.177 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:19:14.177 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:13 smithi132 ceph-mon[66347]: pgmap v195: 161 pgs: 2 active+undersized+degraded, 18 peering, 14 stale+active+clean, 5 active+undersized, 122 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 2/723 objects degraded (0.277%) 2024-05-09T04:19:14.177 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:13 smithi132 podman[81231]: 2024-05-09 04:19:13.827319639 +0000 UTC m=+0.018148466 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:19:14.177 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:13 smithi132 podman[81231]: 2024-05-09 04:19:13.938707573 +0000 UTC m=+0.129536383 container create 448e808a5700488de602cad884af5a3089c6d52b7cf96472a751e67ea080a567 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:19:14.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:14.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:19:14.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:13 smithi078 ceph-mon[83176]: pgmap v195: 161 pgs: 2 active+undersized+degraded, 18 peering, 14 stale+active+clean, 5 active+undersized, 122 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 2/723 objects degraded (0.277%) 2024-05-09T04:19:14.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:14.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:19:14.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:13 smithi078 ceph-mon[88406]: pgmap v195: 161 pgs: 2 active+undersized+degraded, 18 peering, 14 stale+active+clean, 5 active+undersized, 122 active+clean; 585 KiB data, 182 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 2/723 objects degraded (0.277%) 2024-05-09T04:19:14.503 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 podman[81231]: 2024-05-09 04:19:14.291715453 +0000 UTC m=+0.482544263 container init 448e808a5700488de602cad884af5a3089c6d52b7cf96472a751e67ea080a567 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:19:14.503 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 podman[81231]: 2024-05-09 04:19:14.295241785 +0000 UTC m=+0.486070596 container start 448e808a5700488de602cad884af5a3089c6d52b7cf96472a751e67ea080a567 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-05-09T04:19:14.503 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 podman[81231]: 2024-05-09 04:19:14.374945667 +0000 UTC m=+0.565774491 container attach 448e808a5700488de602cad884af5a3089c6d52b7cf96472a751e67ea080a567 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-05-09T04:19:15.094 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:14 smithi132 ceph-mon[66347]: Health check failed: Degraded data redundancy: 2/723 objects degraded (0.277%), 2 pgs degraded (PG_DEGRADED) 2024-05-09T04:19:15.095 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:14 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:15.095 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate[81246]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:19:15.095 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate[81246]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:19:15.095 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 bash[81231]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:19:15.095 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 bash[81231]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:19:15.095 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 bash[81231]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-09T04:19:15.095 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 bash[81231]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-09T04:19:15.095 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 bash[81231]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-09T04:19:15.095 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 bash[81231]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-09T04:19:15.095 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 bash[81231]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-05-09T04:19:15.095 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 bash[81231]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-09T04:19:15.096 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 bash[81231]: --> ceph-volume raw activate successful for osd ID: 5 2024-05-09T04:19:15.096 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate[81246]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-09T04:19:15.096 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate[81246]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-09T04:19:15.096 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate[81246]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-09T04:19:15.096 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate[81246]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-09T04:19:15.096 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate[81246]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-05-09T04:19:15.096 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate[81246]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-09T04:19:15.096 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate[81246]: --> ceph-volume raw activate successful for osd ID: 5 2024-05-09T04:19:15.096 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:14 smithi132 podman[81231]: 2024-05-09 04:19:14.802494693 +0000 UTC m=+0.993323508 container died 448e808a5700488de602cad884af5a3089c6d52b7cf96472a751e67ea080a567 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:19:15.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:14 smithi078 ceph-mon[83176]: Health check failed: Degraded data redundancy: 2/723 objects degraded (0.277%), 2 pgs degraded (PG_DEGRADED) 2024-05-09T04:19:15.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:14 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:15.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:14 smithi078 ceph-mon[88406]: Health check failed: Degraded data redundancy: 2/723 objects degraded (0.277%), 2 pgs degraded (PG_DEGRADED) 2024-05-09T04:19:15.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:14 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:15.751 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:15 smithi132 podman[81383]: 2024-05-09 04:19:15.378950646 +0000 UTC m=+0.570961813 container remove 448e808a5700488de602cad884af5a3089c6d52b7cf96472a751e67ea080a567 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:19:15.751 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:15 smithi132 podman[81448]: 2024-05-09 04:19:15.499064797 +0000 UTC m=+0.018485517 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:19:15.751 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:15 smithi132 podman[81448]: 2024-05-09 04:19:15.605092739 +0000 UTC m=+0.124513488 container create 408a908ef3cc5287f942c38c9fff401b0c24f2a3c4baee3c768d94d25966e85e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240507, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS) 2024-05-09T04:19:16.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:15 smithi132 ceph-mon[66347]: pgmap v196: 161 pgs: 16 active+undersized+degraded, 18 peering, 25 active+undersized, 102 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-05-09T04:19:16.253 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:15 smithi132 podman[81448]: 2024-05-09 04:19:15.857748901 +0000 UTC m=+0.377169608 container init 408a908ef3cc5287f942c38c9fff401b0c24f2a3c4baee3c768d94d25966e85e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, ceph=True) 2024-05-09T04:19:16.253 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:15 smithi132 podman[81448]: 2024-05-09 04:19:15.860962107 +0000 UTC m=+0.380382817 container start 408a908ef3cc5287f942c38c9fff401b0c24f2a3c4baee3c768d94d25966e85e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.build-date=20240507, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-05-09T04:19:16.253 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:15 smithi132 ceph-osd[81467]: -- 172.21.15.132:0/156575574 <== mon.2 v2:172.21.15.132:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5563ddc57680 con 0x5563ddc21000 2024-05-09T04:19:16.253 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:15 smithi132 bash[81448]: 408a908ef3cc5287f942c38c9fff401b0c24f2a3c4baee3c768d94d25966e85e 2024-05-09T04:19:16.253 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:15 smithi132 systemd[1]: Started Ceph osd.5 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:19:16.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:15 smithi078 ceph-mon[83176]: pgmap v196: 161 pgs: 16 active+undersized+degraded, 18 peering, 25 active+undersized, 102 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-05-09T04:19:16.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:15 smithi078 ceph-mon[88406]: pgmap v196: 161 pgs: 16 active+undersized+degraded, 18 peering, 25 active+undersized, 102 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-05-09T04:19:16.753 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:16 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:19:16.416+0000 7fd2bc9817c0 -1 Falling back to public interface 2024-05-09T04:19:17.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:17 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:17.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:17 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:17.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:17 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:19:17.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:17 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:17.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:17 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:17.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:17 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:19:17.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:17 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:17.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:17 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:17.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:17 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:19:18.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:18 smithi078 ceph-mon[83176]: pgmap v197: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-05-09T04:19:18.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:18 smithi078 ceph-mon[83176]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-05-09T04:19:18.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:18 smithi078 ceph-mon[88406]: pgmap v197: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-05-09T04:19:18.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:18 smithi078 ceph-mon[88406]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-05-09T04:19:18.342 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:18 smithi132 ceph-mon[66347]: pgmap v197: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-05-09T04:19:18.342 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:18 smithi132 ceph-mon[66347]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-05-09T04:19:19.503 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:19:19.120+0000 7fd2bc9817c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-05-09T04:19:19.503 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:19:19.147+0000 7fd2bc9817c0 -1 osd.5 138 log_to_monitors true 2024-05-09T04:19:19.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:19:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:19:19] "GET /metrics HTTP/1.1" 200 38342 "" "Prometheus/2.43.0" 2024-05-09T04:19:20.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:20 smithi132 ceph-mon[66347]: pgmap v198: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-05-09T04:19:20.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:20 smithi132 ceph-mon[66347]: from='osd.5 [v2:172.21.15.132:6808/562889100,v1:172.21.15.132:6809/562889100]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-09T04:19:20.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:20 smithi132 ceph-mon[66347]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-09T04:19:20.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:20 smithi078 ceph-mon[83176]: pgmap v198: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-05-09T04:19:20.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:20 smithi078 ceph-mon[83176]: from='osd.5 [v2:172.21.15.132:6808/562889100,v1:172.21.15.132:6809/562889100]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-09T04:19:20.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:20 smithi078 ceph-mon[83176]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-09T04:19:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:20 smithi078 ceph-mon[88406]: pgmap v198: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-05-09T04:19:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:20 smithi078 ceph-mon[88406]: from='osd.5 [v2:172.21.15.132:6808/562889100,v1:172.21.15.132:6809/562889100]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-09T04:19:20.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:20 smithi078 ceph-mon[88406]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-09T04:19:21.003 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:19:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:19:20.506+0000 7fd2a9dd0700 -1 osd.5 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T04:19:21.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:21 smithi132 ceph-mon[66347]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-09T04:19:21.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:21 smithi132 ceph-mon[66347]: osdmap e141: 8 total, 7 up, 8 in 2024-05-09T04:19:21.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:21 smithi132 ceph-mon[66347]: from='osd.5 [v2:172.21.15.132:6808/562889100,v1:172.21.15.132:6809/562889100]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:19:21.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:21 smithi132 ceph-mon[66347]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:19:21.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:21 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:21.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:21 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:21.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:21 smithi132 ceph-mon[66347]: from='osd.5 ' entity='osd.5' 2024-05-09T04:19:21.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[88406]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-09T04:19:21.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[88406]: osdmap e141: 8 total, 7 up, 8 in 2024-05-09T04:19:21.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[88406]: from='osd.5 [v2:172.21.15.132:6808/562889100,v1:172.21.15.132:6809/562889100]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:19:21.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[88406]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:19:21.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:21.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:21.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[88406]: from='osd.5 ' entity='osd.5' 2024-05-09T04:19:21.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[83176]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-09T04:19:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[83176]: osdmap e141: 8 total, 7 up, 8 in 2024-05-09T04:19:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[83176]: from='osd.5 [v2:172.21.15.132:6808/562889100,v1:172.21.15.132:6809/562889100]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:19:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[83176]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:19:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:21.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:21 smithi078 ceph-mon[83176]: from='osd.5 ' entity='osd.5' 2024-05-09T04:19:22.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:22 smithi132 ceph-mon[66347]: pgmap v200: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-05-09T04:19:22.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:22 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:22.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:22 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:22.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:22 smithi132 ceph-mon[66347]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:19:22.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:22 smithi132 ceph-mon[66347]: osd.5 [v2:172.21.15.132:6808/562889100,v1:172.21.15.132:6809/562889100] boot 2024-05-09T04:19:22.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:22 smithi132 ceph-mon[66347]: osdmap e142: 8 total, 8 up, 8 in 2024-05-09T04:19:22.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:22 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:19:22.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[88406]: pgmap v200: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-05-09T04:19:22.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:22.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:22.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[88406]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:19:22.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[88406]: osd.5 [v2:172.21.15.132:6808/562889100,v1:172.21.15.132:6809/562889100] boot 2024-05-09T04:19:22.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[88406]: osdmap e142: 8 total, 8 up, 8 in 2024-05-09T04:19:22.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:19:22.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[83176]: pgmap v200: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-05-09T04:19:22.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:22.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:22.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[83176]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:19:22.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[83176]: osd.5 [v2:172.21.15.132:6808/562889100,v1:172.21.15.132:6809/562889100] boot 2024-05-09T04:19:22.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[83176]: osdmap e142: 8 total, 8 up, 8 in 2024-05-09T04:19:22.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:22 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-09T04:19:23.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:23 smithi132 ceph-mon[66347]: osdmap e143: 8 total, 8 up, 8 in 2024-05-09T04:19:23.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:23 smithi132 ceph-mon[66347]: Health check update: Degraded data redundancy: 80/723 objects degraded (11.065%), 20 pgs degraded (PG_DEGRADED) 2024-05-09T04:19:24.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:23 smithi078 ceph-mon[83176]: osdmap e143: 8 total, 8 up, 8 in 2024-05-09T04:19:24.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:23 smithi078 ceph-mon[83176]: Health check update: Degraded data redundancy: 80/723 objects degraded (11.065%), 20 pgs degraded (PG_DEGRADED) 2024-05-09T04:19:24.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:23 smithi078 ceph-mon[88406]: osdmap e143: 8 total, 8 up, 8 in 2024-05-09T04:19:24.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:23 smithi078 ceph-mon[88406]: Health check update: Degraded data redundancy: 80/723 objects degraded (11.065%), 20 pgs degraded (PG_DEGRADED) 2024-05-09T04:19:24.943 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:24 smithi132 ceph-mon[66347]: pgmap v203: 161 pgs: 18 active+undersized+degraded, 36 active+undersized, 107 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 78/723 objects degraded (10.788%) 2024-05-09T04:19:25.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:24 smithi078 ceph-mon[83176]: pgmap v203: 161 pgs: 18 active+undersized+degraded, 36 active+undersized, 107 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 78/723 objects degraded (10.788%) 2024-05-09T04:19:25.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:24 smithi078 ceph-mon[88406]: pgmap v203: 161 pgs: 18 active+undersized+degraded, 36 active+undersized, 107 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 78/723 objects degraded (10.788%) 2024-05-09T04:19:25.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:19:25 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:19:25.157+0000 7f81013d6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-05-09T04:19:26.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:25 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:19:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:19:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:19:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:19:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:19:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:19:26.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:25 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-09T04:19:26.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:26.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:26.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:26.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:19:26.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:19:26.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:26.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:19:26.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:19:26.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:19:26.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:25 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-09T04:19:27.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:26 smithi132 ceph-mon[66347]: pgmap v204: 161 pgs: 11 active+undersized+degraded, 23 active+undersized, 127 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 56/723 objects degraded (7.746%) 2024-05-09T04:19:27.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:26 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-09T04:19:27.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:26 smithi132 ceph-mon[66347]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-05-09T04:19:27.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:26 smithi078 ceph-mon[83176]: pgmap v204: 161 pgs: 11 active+undersized+degraded, 23 active+undersized, 127 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 56/723 objects degraded (7.746%) 2024-05-09T04:19:27.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:26 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-09T04:19:27.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:26 smithi078 ceph-mon[83176]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-05-09T04:19:27.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:26 smithi078 ceph-mon[88406]: pgmap v204: 161 pgs: 11 active+undersized+degraded, 23 active+undersized, 127 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 56/723 objects degraded (7.746%) 2024-05-09T04:19:27.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:26 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-09T04:19:27.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:26 smithi078 ceph-mon[88406]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-05-09T04:19:28.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:27 smithi132 ceph-mon[66347]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 11 pgs degraded) 2024-05-09T04:19:28.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:27 smithi132 ceph-mon[66347]: Cluster is now healthy 2024-05-09T04:19:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:27 smithi078 ceph-mon[83176]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 11 pgs degraded) 2024-05-09T04:19:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:27 smithi078 ceph-mon[83176]: Cluster is now healthy 2024-05-09T04:19:28.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:27 smithi078 ceph-mon[88406]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 11 pgs degraded) 2024-05-09T04:19:28.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:27 smithi078 ceph-mon[88406]: Cluster is now healthy 2024-05-09T04:19:28.915 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:28 smithi078 ceph-mon[88406]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 440 B/s rd, 0 op/s 2024-05-09T04:19:28.916 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:28.916 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:19:28.916 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:28 smithi078 ceph-mon[83176]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 440 B/s rd, 0 op/s 2024-05-09T04:19:28.916 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:28.916 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:19:28.916 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:19:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:28 smithi132 ceph-mon[66347]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 440 B/s rd, 0 op/s 2024-05-09T04:19:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:19:29.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:19:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:19:29] "GET /metrics HTTP/1.1" 200 38357 "" "Prometheus/2.43.0" 2024-05-09T04:19:29.584 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:19:29.584 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (8m) 80s ago 30m 28.0M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:19:29.584 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (7m) 9s ago 29m 78.3M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:19:29.584 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (9m) 80s ago 29m 67.2M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:19:29.584 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (9m) 9s ago 34m 468M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (19m) 80s ago 36m 524M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (5m) 80s ago 36m 51.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (6m) 9s ago 34m 46.8M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (5m) 80s ago 35m 39.8M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (8m) 80s ago 30m 16.6M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (8m) 9s ago 30m 16.5M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (3m) 80s ago 34m 48.0M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 80s ago 33m 46.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (2m) 80s ago 33m 40.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (85s) 80s ago 33m 12.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a cfe832ae10c3 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (49s) 9s ago 32m 44.6M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a cdf9ea05dcb0 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (14s) 9s ago 32m 12.5M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 408a908ef3cc 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (32m) 9s ago 32m 58.0M 3248M 17.2.0 e1d6a67b021e e06fde1c5a00 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (31m) 9s ago 31m 62.4M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:19:29.585 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (8m) 9s ago 30m 70.0M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:19:29.586 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (29m) 80s ago 29m 80.2M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:19:29.586 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (29m) 9s ago 29m 82.1M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:19:29.990 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:19:29.990 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 6 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:19:29.991 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:19:29.992 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-05-09T04:19:29.992 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 11 2024-05-09T04:19:29.992 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:19:29.992 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:19:30.334 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:19:30.334 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:19:30.334 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:19:30.334 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-09T04:19:30.334 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:19:30.334 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "6/8 daemons upgraded", 2024-05-09T04:19:30.335 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-05-09T04:19:30.335 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:19:30.335 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:19:30.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:30 smithi078 ceph-mon[88406]: from='client.54481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:30.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:30 smithi078 ceph-mon[88406]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:19:30.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:30 smithi078 ceph-mon[88406]: from='client.44547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:30.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:30 smithi078 ceph-mon[88406]: from='client.54493 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:30.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:30 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/1277326718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:19:30.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:30 smithi078 ceph-mon[83176]: from='client.54481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:30.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:30 smithi078 ceph-mon[83176]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:19:30.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:30 smithi078 ceph-mon[83176]: from='client.44547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:30.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:30 smithi078 ceph-mon[83176]: from='client.54493 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:30.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:30 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/1277326718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:19:31.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:30 smithi132 ceph-mon[66347]: from='client.54481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:31.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:30 smithi132 ceph-mon[66347]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:19:31.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:30 smithi132 ceph-mon[66347]: from='client.44547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:31.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:30 smithi132 ceph-mon[66347]: from='client.54493 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:31.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:30 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/1277326718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:19:32.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:31 smithi132 ceph-mon[66347]: from='client.54505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:32.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:31 smithi078 ceph-mon[88406]: from='client.54505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:32.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:31 smithi078 ceph-mon[83176]: from='client.54505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:19:33.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:32 smithi132 ceph-mon[66347]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-05-09T04:19:33.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:32 smithi078 ceph-mon[83176]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-05-09T04:19:33.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:32 smithi078 ceph-mon[88406]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-05-09T04:19:34.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:34 smithi132 ceph-mon[66347]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 781 B/s rd, 0 op/s 2024-05-09T04:19:34.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:34.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:34 smithi078 ceph-mon[88406]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 781 B/s rd, 0 op/s 2024-05-09T04:19:34.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:34.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:34 smithi078 ceph-mon[83176]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 781 B/s rd, 0 op/s 2024-05-09T04:19:34.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:35.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:35 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:35.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:35 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:35.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:35 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:36.404 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:36 smithi132 ceph-mon[66347]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-09T04:19:36.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:36 smithi078 ceph-mon[83176]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-09T04:19:36.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:36 smithi078 ceph-mon[88406]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-09T04:19:38.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:38 smithi132 ceph-mon[66347]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:38.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:38 smithi078 ceph-mon[83176]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:38.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:38 smithi078 ceph-mon[88406]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:39.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:19:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:19:39] "GET /metrics HTTP/1.1" 200 38357 "" "Prometheus/2.43.0" 2024-05-09T04:19:40.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:40 smithi132 ceph-mon[66347]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:40.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:40 smithi078 ceph-mon[83176]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:40.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:40 smithi078 ceph-mon[88406]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:41.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:41 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-09T04:19:41.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:41 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-09T04:19:41.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:41 smithi132 ceph-mon[66347]: Upgrade: osd.6 is safe to restart 2024-05-09T04:19:41.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:41 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-09T04:19:41.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:41 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-09T04:19:41.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:41 smithi078 ceph-mon[83176]: Upgrade: osd.6 is safe to restart 2024-05-09T04:19:41.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:41 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-09T04:19:41.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:41 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-09T04:19:41.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:41 smithi078 ceph-mon[88406]: Upgrade: osd.6 is safe to restart 2024-05-09T04:19:42.420 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:42 smithi132 ceph-mon[66347]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:42.421 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:42.421 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-09T04:19:42.421 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:19:42.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:42 smithi078 ceph-mon[88406]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:42.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:42 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:42.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:42 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-09T04:19:42.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:42 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:19:42.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:42 smithi078 ceph-mon[83176]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:42.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-09T04:19:42.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:19:43.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:43 smithi132 ceph-mon[66347]: Upgrade: Updating osd.6 2024-05-09T04:19:43.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:43 smithi132 ceph-mon[66347]: Deploying daemon osd.6 on smithi132 2024-05-09T04:19:43.491 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:19:43.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:43 smithi078 ceph-mon[83176]: Upgrade: Updating osd.6 2024-05-09T04:19:43.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:43 smithi078 ceph-mon[83176]: Deploying daemon osd.6 on smithi132 2024-05-09T04:19:43.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:19:43.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:43 smithi078 ceph-mon[88406]: Upgrade: Updating osd.6 2024-05-09T04:19:43.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:43 smithi078 ceph-mon[88406]: Deploying daemon osd.6 on smithi132 2024-05-09T04:19:43.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:19:44.412 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:44 smithi078 ceph-mon[88406]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:44.412 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:44 smithi078 ceph-mon[83176]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:44.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:44 smithi132 ceph-mon[66347]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:19:45.437 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:45 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:45.438 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:45 smithi132 systemd[1]: Stopping Ceph osd.6 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:19:45.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:45 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:45.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:45 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:45.753 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:45 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[33015]: 2024-05-09T04:19:45.437+0000 7f78b04ef700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:19:45.754 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:45 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[33015]: 2024-05-09T04:19:45.437+0000 7f78b04ef700 -1 osd.6 143 *** Got signal Terminated *** 2024-05-09T04:19:45.754 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:45 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[33015]: 2024-05-09T04:19:45.437+0000 7f78b04ef700 -1 osd.6 143 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:19:46.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:46 smithi132 ceph-mon[66347]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:46.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:46 smithi132 ceph-mon[66347]: osd.6 marked itself down and dead 2024-05-09T04:19:46.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:46 smithi078 ceph-mon[83176]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:46.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:46 smithi078 ceph-mon[83176]: osd.6 marked itself down and dead 2024-05-09T04:19:46.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:46 smithi078 ceph-mon[88406]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:19:46.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:46 smithi078 ceph-mon[88406]: osd.6 marked itself down and dead 2024-05-09T04:19:46.988 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:46 smithi132 podman[85799]: 2024-05-09 04:19:46.531002388 +0000 UTC m=+1.194556133 container died e06fde1c5a00ccc3aa8fd6b2b0010439a0fe8d399b721442c31e816fec7bcffa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6, architecture=x86_64, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, RELEASE=HEAD, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, version=8, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870) 2024-05-09T04:19:47.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:47 smithi132 ceph-mon[66347]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:19:47.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:47 smithi132 ceph-mon[66347]: osdmap e144: 8 total, 7 up, 8 in 2024-05-09T04:19:47.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:47 smithi132 podman[85799]: 2024-05-09 04:19:47.070650644 +0000 UTC m=+1.734204385 container cleanup e06fde1c5a00ccc3aa8fd6b2b0010439a0fe8d399b721442c31e816fec7bcffa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6, vcs-type=git, GIT_CLEAN=True, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, build-date=2022-05-03T08:36:31.336870, ceph=True, distribution-scope=public, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_BRANCH=HEAD, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-09T04:19:47.254 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:47 smithi132 bash[85799]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6 2024-05-09T04:19:47.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:47 smithi078 ceph-mon[83176]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:19:47.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:47 smithi078 ceph-mon[83176]: osdmap e144: 8 total, 7 up, 8 in 2024-05-09T04:19:47.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:47 smithi078 ceph-mon[88406]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:19:47.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:47 smithi078 ceph-mon[88406]: osdmap e144: 8 total, 7 up, 8 in 2024-05-09T04:19:47.686 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:47 smithi132 podman[85816]: 2024-05-09 04:19:47.272412864 +0000 UTC m=+0.740615045 container remove e06fde1c5a00ccc3aa8fd6b2b0010439a0fe8d399b721442c31e816fec7bcffa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6, name=centos-stream, GIT_CLEAN=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, version=8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, release=754, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-09T04:19:47.967 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:47 smithi132 podman[85911]: 2024-05-09 04:19:47.589336335 +0000 UTC m=+0.017875098 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:19:47.967 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:47 smithi132 podman[85911]: 2024-05-09 04:19:47.720282314 +0000 UTC m=+0.148821057 container create b29410022d01b176b78434456ba90deb002d7c40c8e3e3e877cf2dced168d9cc (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-05-09T04:19:48.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:48 smithi132 ceph-mon[66347]: pgmap v216: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:19:48.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:48 smithi132 ceph-mon[66347]: osdmap e145: 8 total, 7 up, 8 in 2024-05-09T04:19:48.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:47 smithi132 podman[85911]: 2024-05-09 04:19:47.968080093 +0000 UTC m=+0.396618838 container init b29410022d01b176b78434456ba90deb002d7c40c8e3e3e877cf2dced168d9cc (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0) 2024-05-09T04:19:48.254 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:47 smithi132 podman[85911]: 2024-05-09 04:19:47.971739136 +0000 UTC m=+0.400277885 container start b29410022d01b176b78434456ba90deb002d7c40c8e3e3e877cf2dced168d9cc (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-05-09T04:19:48.254 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:48 smithi132 podman[85911]: 2024-05-09 04:19:48.050978506 +0000 UTC m=+0.479517250 container attach b29410022d01b176b78434456ba90deb002d7c40c8e3e3e877cf2dced168d9cc (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-05-09T04:19:48.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:48 smithi078 ceph-mon[83176]: pgmap v216: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:19:48.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:48 smithi078 ceph-mon[83176]: osdmap e145: 8 total, 7 up, 8 in 2024-05-09T04:19:48.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:48 smithi078 ceph-mon[88406]: pgmap v216: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:19:48.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:48 smithi078 ceph-mon[88406]: osdmap e145: 8 total, 7 up, 8 in 2024-05-09T04:19:48.561 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:48 smithi132 podman[85911]: 2024-05-09 04:19:48.267761755 +0000 UTC m=+0.696300493 container died b29410022d01b176b78434456ba90deb002d7c40c8e3e3e877cf2dced168d9cc (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-05-09T04:19:49.182 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:48 smithi132 podman[85936]: 2024-05-09 04:19:48.871735902 +0000 UTC m=+0.597807033 container remove b29410022d01b176b78434456ba90deb002d7c40c8e3e3e877cf2dced168d9cc (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5) 2024-05-09T04:19:49.183 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:48 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.6.service: Deactivated successfully. 2024-05-09T04:19:49.183 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:48 smithi132 systemd[1]: Stopped Ceph osd.6 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:19:49.183 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:48 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.6.service: Consumed 14.709s CPU time. 2024-05-09T04:19:49.183 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:49 smithi132 systemd[1]: Starting Ceph osd.6 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:19:49.450 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:49 smithi132 ceph-mon[66347]: Health check failed: Degraded data redundancy: 32/723 objects degraded (4.426%), 9 pgs degraded (PG_DEGRADED) 2024-05-09T04:19:49.451 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:49 smithi132 podman[86042]: 2024-05-09 04:19:49.214881945 +0000 UTC m=+0.018371287 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:19:49.451 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:49 smithi132 podman[86042]: 2024-05-09 04:19:49.331395732 +0000 UTC m=+0.134885097 container create 3d560f8e3651661ff177d6ce556ec7597f30246323eef1443d65dc4ddf12a147 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-05-09T04:19:49.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:49 smithi078 ceph-mon[83176]: Health check failed: Degraded data redundancy: 32/723 objects degraded (4.426%), 9 pgs degraded (PG_DEGRADED) 2024-05-09T04:19:49.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:49 smithi078 ceph-mon[88406]: Health check failed: Degraded data redundancy: 32/723 objects degraded (4.426%), 9 pgs degraded (PG_DEGRADED) 2024-05-09T04:19:49.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:19:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:19:49] "GET /metrics HTTP/1.1" 200 38358 "" "Prometheus/2.43.0" 2024-05-09T04:19:49.753 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:49 smithi132 podman[86042]: 2024-05-09 04:19:49.525809203 +0000 UTC m=+0.329298530 container init 3d560f8e3651661ff177d6ce556ec7597f30246323eef1443d65dc4ddf12a147 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True) 2024-05-09T04:19:49.753 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:49 smithi132 podman[86042]: 2024-05-09 04:19:49.529345236 +0000 UTC m=+0.332834567 container start 3d560f8e3651661ff177d6ce556ec7597f30246323eef1443d65dc4ddf12a147 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-05-09T04:19:49.753 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:49 smithi132 podman[86042]: 2024-05-09 04:19:49.600256192 +0000 UTC m=+0.403745517 container attach 3d560f8e3651661ff177d6ce556ec7597f30246323eef1443d65dc4ddf12a147 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-05-09T04:19:50.354 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:50 smithi132 ceph-mon[66347]: pgmap v218: 161 pgs: 9 active+undersized+degraded, 11 stale+active+clean, 12 active+undersized, 129 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-05-09T04:19:50.354 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate[86057]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:19:50.354 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate[86057]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:19:50.354 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 bash[86042]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 bash[86042]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 bash[86042]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 bash[86042]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 bash[86042]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 bash[86042]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 bash[86042]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 bash[86042]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 bash[86042]: --> ceph-volume raw activate successful for osd ID: 6 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate[86057]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate[86057]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate[86057]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate[86057]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate[86057]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-05-09T04:19:50.355 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate[86057]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-09T04:19:50.356 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate[86057]: --> ceph-volume raw activate successful for osd ID: 6 2024-05-09T04:19:50.356 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 podman[86042]: 2024-05-09 04:19:50.040854909 +0000 UTC m=+0.844344224 container died 3d560f8e3651661ff177d6ce556ec7597f30246323eef1443d65dc4ddf12a147 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507) 2024-05-09T04:19:50.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:50 smithi078 ceph-mon[83176]: pgmap v218: 161 pgs: 9 active+undersized+degraded, 11 stale+active+clean, 12 active+undersized, 129 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-05-09T04:19:50.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:50 smithi078 ceph-mon[88406]: pgmap v218: 161 pgs: 9 active+undersized+degraded, 11 stale+active+clean, 12 active+undersized, 129 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-05-09T04:19:50.946 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 podman[86194]: 2024-05-09 04:19:50.646071544 +0000 UTC m=+0.596334394 container remove 3d560f8e3651661ff177d6ce556ec7597f30246323eef1443d65dc4ddf12a147 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:19:50.946 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 podman[86262]: 2024-05-09 04:19:50.767819703 +0000 UTC m=+0.018593132 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:19:50.946 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:50 smithi132 podman[86262]: 2024-05-09 04:19:50.947284319 +0000 UTC m=+0.198057733 container create 01f14aea01f3e351ffb9ea3d84ff4e9e0a20d14e47d6890ba5512200abca1336 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:19:51.232 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:51 smithi132 podman[86262]: 2024-05-09 04:19:51.14114107 +0000 UTC m=+0.391914495 container init 01f14aea01f3e351ffb9ea3d84ff4e9e0a20d14e47d6890ba5512200abca1336 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_CLEAN=True, org.label-schema.build-date=20240507) 2024-05-09T04:19:51.232 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:51 smithi132 podman[86262]: 2024-05-09 04:19:51.144234535 +0000 UTC m=+0.395007955 container start 01f14aea01f3e351ffb9ea3d84ff4e9e0a20d14e47d6890ba5512200abca1336 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2, ceph=True) 2024-05-09T04:19:51.232 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:51 smithi132 ceph-osd[86281]: -- 172.21.15.132:0/1292994918 <== mon.2 v2:172.21.15.132:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f9ac259680 con 0x55f9ac222c00 2024-05-09T04:19:51.503 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:51 smithi132 bash[86262]: 01f14aea01f3e351ffb9ea3d84ff4e9e0a20d14e47d6890ba5512200abca1336 2024-05-09T04:19:51.503 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:51 smithi132 systemd[1]: Started Ceph osd.6 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:19:52.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:19:51.692+0000 7f1d1a7aa7c0 -1 Falling back to public interface 2024-05-09T04:19:52.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:52 smithi132 ceph-mon[66347]: pgmap v219: 161 pgs: 18 active+undersized+degraded, 2 stale+active+clean, 32 active+undersized, 109 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-05-09T04:19:52.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:52 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:52.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:52 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:52.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:52 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:19:52.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:52 smithi078 ceph-mon[83176]: pgmap v219: 161 pgs: 18 active+undersized+degraded, 2 stale+active+clean, 32 active+undersized, 109 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-05-09T04:19:52.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:52 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:52.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:52 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:52.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:52 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:19:52.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:52 smithi078 ceph-mon[88406]: pgmap v219: 161 pgs: 18 active+undersized+degraded, 2 stale+active+clean, 32 active+undersized, 109 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-05-09T04:19:52.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:52 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:52.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:52 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:52.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:52 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:19:54.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:54 smithi132 ceph-mon[66347]: pgmap v220: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 87/723 objects degraded (12.033%) 2024-05-09T04:19:54.503 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:19:54.412+0000 7f1d1a7aa7c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-05-09T04:19:54.504 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:54 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:19:54.444+0000 7f1d1a7aa7c0 -1 osd.6 143 log_to_monitors true 2024-05-09T04:19:54.510 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:54 smithi078 ceph-mon[83176]: pgmap v220: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 87/723 objects degraded (12.033%) 2024-05-09T04:19:54.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:54 smithi078 ceph-mon[88406]: pgmap v220: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 87/723 objects degraded (12.033%) 2024-05-09T04:19:55.503 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:19:55 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:19:55.323+0000 7f1d0bc01700 -1 osd.6 143 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T04:19:55.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:55 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:55.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:55 smithi132 ceph-mon[66347]: from='osd.6 [v2:172.21.15.132:6816/216294402,v1:172.21.15.132:6817/216294402]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-09T04:19:55.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:55 smithi132 ceph-mon[66347]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-09T04:19:55.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:55 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:55.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:55 smithi078 ceph-mon[88406]: from='osd.6 [v2:172.21.15.132:6816/216294402,v1:172.21.15.132:6817/216294402]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-09T04:19:55.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:55 smithi078 ceph-mon[88406]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-09T04:19:55.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:55 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:19:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:55 smithi078 ceph-mon[83176]: from='osd.6 [v2:172.21.15.132:6816/216294402,v1:172.21.15.132:6817/216294402]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-09T04:19:55.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:55 smithi078 ceph-mon[83176]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-09T04:19:56.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:56 smithi132 ceph-mon[66347]: pgmap v221: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 87/723 objects degraded (12.033%) 2024-05-09T04:19:56.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:56 smithi132 ceph-mon[66347]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-09T04:19:56.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:56 smithi132 ceph-mon[66347]: from='osd.6 [v2:172.21.15.132:6816/216294402,v1:172.21.15.132:6817/216294402]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:19:56.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:56 smithi132 ceph-mon[66347]: osdmap e146: 8 total, 7 up, 8 in 2024-05-09T04:19:56.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:56 smithi132 ceph-mon[66347]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:19:56.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:56 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:56.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:56 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:56.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[83176]: pgmap v221: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 87/723 objects degraded (12.033%) 2024-05-09T04:19:56.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[83176]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-09T04:19:56.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[83176]: from='osd.6 [v2:172.21.15.132:6816/216294402,v1:172.21.15.132:6817/216294402]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:19:56.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[83176]: osdmap e146: 8 total, 7 up, 8 in 2024-05-09T04:19:56.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[83176]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:19:56.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:56.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:56.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[88406]: pgmap v221: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 202 MiB used, 715 GiB / 715 GiB avail; 87/723 objects degraded (12.033%) 2024-05-09T04:19:56.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[88406]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-09T04:19:56.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[88406]: from='osd.6 [v2:172.21.15.132:6816/216294402,v1:172.21.15.132:6817/216294402]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:19:56.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[88406]: osdmap e146: 8 total, 7 up, 8 in 2024-05-09T04:19:56.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[88406]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:19:56.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:56.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:56 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:57.582 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:57 smithi132 ceph-mon[66347]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:19:57.582 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:57 smithi132 ceph-mon[66347]: osd.6 [v2:172.21.15.132:6816/216294402,v1:172.21.15.132:6817/216294402] boot 2024-05-09T04:19:57.582 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:57 smithi132 ceph-mon[66347]: osdmap e147: 8 total, 8 up, 8 in 2024-05-09T04:19:57.582 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:57 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:19:57.583 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:57 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:57.583 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:57 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:57.583 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:57 smithi132 ceph-mon[66347]: osdmap e148: 8 total, 8 up, 8 in 2024-05-09T04:19:57.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[88406]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:19:57.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[88406]: osd.6 [v2:172.21.15.132:6816/216294402,v1:172.21.15.132:6817/216294402] boot 2024-05-09T04:19:57.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[88406]: osdmap e147: 8 total, 8 up, 8 in 2024-05-09T04:19:57.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:19:57.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:57.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:57.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[88406]: osdmap e148: 8 total, 8 up, 8 in 2024-05-09T04:19:57.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[83176]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:19:57.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[83176]: osd.6 [v2:172.21.15.132:6816/216294402,v1:172.21.15.132:6817/216294402] boot 2024-05-09T04:19:57.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[83176]: osdmap e147: 8 total, 8 up, 8 in 2024-05-09T04:19:57.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-09T04:19:57.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:57.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:57.763 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:57 smithi078 ceph-mon[83176]: osdmap e148: 8 total, 8 up, 8 in 2024-05-09T04:19:58.584 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:58 smithi132 ceph-mon[66347]: pgmap v224: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 87/723 objects degraded (12.033%) 2024-05-09T04:19:58.584 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:58 smithi132 ceph-mon[66347]: Health check update: Degraded data redundancy: 87/723 objects degraded (12.033%), 19 pgs degraded (PG_DEGRADED) 2024-05-09T04:19:58.584 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:58.584 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:19:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:19:58.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:58 smithi078 ceph-mon[83176]: pgmap v224: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 87/723 objects degraded (12.033%) 2024-05-09T04:19:58.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:58 smithi078 ceph-mon[83176]: Health check update: Degraded data redundancy: 87/723 objects degraded (12.033%), 19 pgs degraded (PG_DEGRADED) 2024-05-09T04:19:58.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:58.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:19:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:19:58.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:58 smithi078 ceph-mon[88406]: pgmap v224: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 87/723 objects degraded (12.033%) 2024-05-09T04:19:58.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:58 smithi078 ceph-mon[88406]: Health check update: Degraded data redundancy: 87/723 objects degraded (12.033%), 19 pgs degraded (PG_DEGRADED) 2024-05-09T04:19:58.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:19:58.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:19:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:19:59.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:19:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:19:59] "GET /metrics HTTP/1.1" 200 38223 "" "Prometheus/2.43.0" 2024-05-09T04:20:00.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: pgmap v226: 161 pgs: 11 active+undersized+degraded, 23 active+undersized, 127 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 56/723 objects degraded (7.746%) 2024-05-09T04:20:00.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: Health detail: HEALTH_WARN Degraded data redundancy: 56/723 objects degraded (7.746%), 11 pgs degraded 2024-05-09T04:20:00.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: [WRN] PG_DEGRADED: Degraded data redundancy: 56/723 objects degraded (7.746%), 11 pgs degraded 2024-05-09T04:20:00.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-05-09T04:20:00.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-05-09T04:20:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-05-09T04:20:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-05-09T04:20:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: pg 3.a is active+undersized+degraded, acting [4,1] 2024-05-09T04:20:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: pg 3.c is active+undersized+degraded, acting [5,3] 2024-05-09T04:20:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-05-09T04:20:00.504 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-05-09T04:20:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-05-09T04:20:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: pg 4.e is active+undersized+degraded, acting [4,2] 2024-05-09T04:20:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: pg 4.1f is active+undersized+degraded, acting [5,1] 2024-05-09T04:20:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:00.505 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:20:00.506 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:00 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:00.692 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:20:00.692 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:20:00 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:20:00.414+0000 7f81013d6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-05-09T04:20:00.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: pgmap v226: 161 pgs: 11 active+undersized+degraded, 23 active+undersized, 127 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 56/723 objects degraded (7.746%) 2024-05-09T04:20:00.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: Health detail: HEALTH_WARN Degraded data redundancy: 56/723 objects degraded (7.746%), 11 pgs degraded 2024-05-09T04:20:00.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: [WRN] PG_DEGRADED: Degraded data redundancy: 56/723 objects degraded (7.746%), 11 pgs degraded 2024-05-09T04:20:00.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-05-09T04:20:00.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-05-09T04:20:00.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-05-09T04:20:00.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-05-09T04:20:00.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: pg 3.a is active+undersized+degraded, acting [4,1] 2024-05-09T04:20:00.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: pg 3.c is active+undersized+degraded, acting [5,3] 2024-05-09T04:20:00.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-05-09T04:20:00.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-05-09T04:20:00.694 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-05-09T04:20:00.694 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: pg 4.e is active+undersized+degraded, acting [4,2] 2024-05-09T04:20:00.694 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: pg 4.1f is active+undersized+degraded, acting [5,1] 2024-05-09T04:20:00.694 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:00.694 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:00.694 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:00.694 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:20:00.694 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:00.694 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: pgmap v226: 161 pgs: 11 active+undersized+degraded, 23 active+undersized, 127 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 56/723 objects degraded (7.746%) 2024-05-09T04:20:00.694 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: Health detail: HEALTH_WARN Degraded data redundancy: 56/723 objects degraded (7.746%), 11 pgs degraded 2024-05-09T04:20:00.694 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: [WRN] PG_DEGRADED: Degraded data redundancy: 56/723 objects degraded (7.746%), 11 pgs degraded 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: pg 3.a is active+undersized+degraded, acting [4,1] 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: pg 3.c is active+undersized+degraded, acting [5,3] 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: pg 4.e is active+undersized+degraded, acting [4,2] 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: pg 4.1f is active+undersized+degraded, acting [5,1] 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:00.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:00.696 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:20:00.696 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:00 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:01.362 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:20:01.362 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (8m) 111s ago 30m 28.0M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:20:01.362 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (8m) 5s ago 30m 80.4M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:20:01.362 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (9m) 111s ago 29m 67.2M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:20:01.362 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (9m) 5s ago 35m 468M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:20:01.362 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (19m) 111s ago 36m 524M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:20:01.362 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (6m) 111s ago 37m 51.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:20:01.362 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (7m) 5s ago 35m 48.3M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (6m) 111s ago 35m 39.8M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (9m) 111s ago 31m 16.6M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (9m) 5s ago 31m 16.6M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 111s ago 34m 48.0M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 111s ago 34m 46.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (2m) 111s ago 34m 40.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (116s) 111s ago 33m 12.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a cfe832ae10c3 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (81s) 5s ago 33m 46.1M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a cdf9ea05dcb0 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (45s) 5s ago 32m 50.0M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 408a908ef3cc 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (10s) 5s ago 32m 12.5M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 01f14aea01f3 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (32m) 5s ago 32m 63.6M 3248M 17.2.0 e1d6a67b021e f8e23f889c19 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (8m) 5s ago 31m 70.0M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (29m) 111s ago 29m 80.2M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:20:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (29m) 5s ago 29m 82.3M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:20:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:01 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:01 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:01 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:01 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:01 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-09T04:20:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:01 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-09T04:20:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:01 smithi132 ceph-mon[66347]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-05-09T04:20:01.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:01 smithi132 ceph-mon[66347]: from='client.54514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:01.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:01.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:01.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:01.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:01.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-09T04:20:01.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-09T04:20:01.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[88406]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-05-09T04:20:01.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[88406]: from='client.54514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:01.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-09T04:20:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-09T04:20:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[83176]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-05-09T04:20:01.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:01 smithi078 ceph-mon[83176]: from='client.54514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:01.766 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:20:01.766 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:20:01.766 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:20:01.766 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:20:01.766 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 7 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 12 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:20:01.767 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:20:02.108 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:20:02.108 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:20:02.108 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:20:02.108 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-09T04:20:02.108 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:20:02.108 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "7/8 daemons upgraded", 2024-05-09T04:20:02.109 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-05-09T04:20:02.109 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:20:02.109 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:20:02.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:02 smithi078 ceph-mon[83176]: pgmap v227: 161 pgs: 1 active+undersized, 160 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-09T04:20:02.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:02 smithi078 ceph-mon[83176]: from='client.54520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:02.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:02 smithi078 ceph-mon[83176]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 11 pgs degraded) 2024-05-09T04:20:02.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:02 smithi078 ceph-mon[83176]: Cluster is now healthy 2024-05-09T04:20:02.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:02 smithi078 ceph-mon[83176]: from='client.54526 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:02.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:02 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/3709979574' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:02.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:02 smithi078 ceph-mon[88406]: pgmap v227: 161 pgs: 1 active+undersized, 160 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-09T04:20:02.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:02 smithi078 ceph-mon[88406]: from='client.54520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:02 smithi078 ceph-mon[88406]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 11 pgs degraded) 2024-05-09T04:20:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:02 smithi078 ceph-mon[88406]: Cluster is now healthy 2024-05-09T04:20:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:02 smithi078 ceph-mon[88406]: from='client.54526 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:02.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:02 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/3709979574' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:02.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:02 smithi132 ceph-mon[66347]: pgmap v227: 161 pgs: 1 active+undersized, 160 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-09T04:20:02.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:02 smithi132 ceph-mon[66347]: from='client.54520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:02.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:02 smithi132 ceph-mon[66347]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 11 pgs degraded) 2024-05-09T04:20:02.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:02 smithi132 ceph-mon[66347]: Cluster is now healthy 2024-05-09T04:20:02.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:02 smithi132 ceph-mon[66347]: from='client.54526 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:02.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:02 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/3709979574' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:03.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:03 smithi132 ceph-mon[66347]: from='client.54538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:03.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:03 smithi078 ceph-mon[83176]: from='client.54538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:03.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:03 smithi078 ceph-mon[88406]: from='client.54538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:04.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:04 smithi078 ceph-mon[83176]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 531 B/s rd, 0 op/s 2024-05-09T04:20:04.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:04 smithi078 ceph-mon[88406]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 531 B/s rd, 0 op/s 2024-05-09T04:20:04.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:04 smithi132 ceph-mon[66347]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 531 B/s rd, 0 op/s 2024-05-09T04:20:05.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:05 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:05.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:05 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:05.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:05 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:06.654 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:06 smithi132 ceph-mon[66347]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-09T04:20:06.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:06 smithi078 ceph-mon[88406]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-09T04:20:06.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:06 smithi078 ceph-mon[83176]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-09T04:20:08.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:08 smithi132 ceph-mon[66347]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-09T04:20:08.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:08 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:08.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:08 smithi078 ceph-mon[83176]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-09T04:20:08.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:08 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:08.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:08 smithi078 ceph-mon[88406]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-09T04:20:08.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:08 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:09.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:20:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:20:09] "GET /metrics HTTP/1.1" 200 38223 "" "Prometheus/2.43.0" 2024-05-09T04:20:10.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:10 smithi132 ceph-mon[66347]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 23 op/s 2024-05-09T04:20:10.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:10 smithi078 ceph-mon[83176]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 23 op/s 2024-05-09T04:20:10.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:10 smithi078 ceph-mon[88406]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 23 op/s 2024-05-09T04:20:12.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:12 smithi132 ceph-mon[66347]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-05-09T04:20:12.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:12 smithi078 ceph-mon[83176]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-05-09T04:20:12.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:12 smithi078 ceph-mon[88406]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-05-09T04:20:14.225 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:14.225 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:20:14.225 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:13 smithi078 ceph-mon[83176]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 0 B/s wr, 79 op/s 2024-05-09T04:20:14.226 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:14.226 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:20:14.226 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:13 smithi078 ceph-mon[88406]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 0 B/s wr, 79 op/s 2024-05-09T04:20:14.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:14.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:20:14.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:13 smithi132 ceph-mon[66347]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 0 B/s wr, 79 op/s 2024-05-09T04:20:15.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:14 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:15.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:14 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:15.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:14 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:16.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:15 smithi132 ceph-mon[66347]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 0 B/s wr, 107 op/s 2024-05-09T04:20:16.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:15 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-09T04:20:16.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:15 smithi132 ceph-mon[66347]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-09T04:20:16.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:15 smithi132 ceph-mon[66347]: Upgrade: osd.7 is safe to restart 2024-05-09T04:20:16.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:15 smithi078 ceph-mon[83176]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 0 B/s wr, 107 op/s 2024-05-09T04:20:16.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:15 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-09T04:20:16.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:15 smithi078 ceph-mon[83176]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-09T04:20:16.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:15 smithi078 ceph-mon[83176]: Upgrade: osd.7 is safe to restart 2024-05-09T04:20:16.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:15 smithi078 ceph-mon[88406]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 0 B/s wr, 107 op/s 2024-05-09T04:20:16.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:15 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-09T04:20:16.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:15 smithi078 ceph-mon[88406]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-09T04:20:16.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:15 smithi078 ceph-mon[88406]: Upgrade: osd.7 is safe to restart 2024-05-09T04:20:18.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:18 smithi132 ceph-mon[66347]: Upgrade: Updating osd.7 2024-05-09T04:20:18.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:18.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-09T04:20:18.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:18 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:18.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:18 smithi132 ceph-mon[66347]: Deploying daemon osd.7 on smithi132 2024-05-09T04:20:18.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:18 smithi132 ceph-mon[66347]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-09T04:20:18.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:18 smithi078 ceph-mon[83176]: Upgrade: Updating osd.7 2024-05-09T04:20:18.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:18.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-09T04:20:18.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:18 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:18.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:18 smithi078 ceph-mon[83176]: Deploying daemon osd.7 on smithi132 2024-05-09T04:20:18.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:18 smithi078 ceph-mon[83176]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-09T04:20:18.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:18 smithi078 ceph-mon[88406]: Upgrade: Updating osd.7 2024-05-09T04:20:18.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:18.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-09T04:20:18.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:18 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:18.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:18 smithi078 ceph-mon[88406]: Deploying daemon osd.7 on smithi132 2024-05-09T04:20:18.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:18 smithi078 ceph-mon[88406]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-09T04:20:19.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:20:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:20:19] "GET /metrics HTTP/1.1" 200 38371 "" "Prometheus/2.43.0" 2024-05-09T04:20:20.222 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:20 smithi132 ceph-mon[66347]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:20:20.493 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:20 smithi132 systemd[1]: Stopping Ceph osd.7 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:20:20.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:20 smithi078 ceph-mon[83176]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:20:20.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:20 smithi078 ceph-mon[88406]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-09T04:20:20.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[35931]: 2024-05-09T04:20:20.493+0000 7f2254d5b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:20:20.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[35931]: 2024-05-09T04:20:20.493+0000 7f2254d5b700 -1 osd.7 148 *** Got signal Terminated *** 2024-05-09T04:20:20.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[35931]: 2024-05-09T04:20:20.493+0000 7f2254d5b700 -1 osd.7 148 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:20:21.494 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:21 smithi132 ceph-mon[66347]: osd.7 marked itself down and dead 2024-05-09T04:20:21.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:21 smithi078 ceph-mon[83176]: osd.7 marked itself down and dead 2024-05-09T04:20:21.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:21 smithi078 ceph-mon[88406]: osd.7 marked itself down and dead 2024-05-09T04:20:21.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:21 smithi132 podman[90346]: 2024-05-09 04:20:21.49546808 +0000 UTC m=+1.113557960 container died f8e23f889c197c5e03f68033d45d949c4a1d8cd1100de3ee5b0adc594d2269d8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.openshift.expose-services=, vcs-type=git, RELEASE=HEAD, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, architecture=x86_64) 2024-05-09T04:20:22.252 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:22 smithi132 ceph-mon[66347]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 98 op/s 2024-05-09T04:20:22.252 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:22 smithi132 ceph-mon[66347]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:20:22.252 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:22 smithi132 ceph-mon[66347]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-09T04:20:22.252 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:22 smithi132 ceph-mon[66347]: osdmap e149: 8 total, 7 up, 8 in 2024-05-09T04:20:22.253 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:21 smithi132 podman[90346]: 2024-05-09 04:20:21.951810932 +0000 UTC m=+1.569900855 container cleanup f8e23f889c197c5e03f68033d45d949c4a1d8cd1100de3ee5b0adc594d2269d8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, vcs-type=git, release=754, distribution-scope=public, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.component=centos-stream-container, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-09T04:20:22.253 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:21 smithi132 bash[90346]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7 2024-05-09T04:20:22.503 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:22 smithi132 podman[90364]: 2024-05-09 04:20:22.253130833 +0000 UTC m=+0.752114012 container remove f8e23f889c197c5e03f68033d45d949c4a1d8cd1100de3ee5b0adc594d2269d8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, name=centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_BRANCH=HEAD) 2024-05-09T04:20:22.510 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:22 smithi078 ceph-mon[88406]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 98 op/s 2024-05-09T04:20:22.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:22 smithi078 ceph-mon[88406]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:20:22.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:22 smithi078 ceph-mon[88406]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-09T04:20:22.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:22 smithi078 ceph-mon[88406]: osdmap e149: 8 total, 7 up, 8 in 2024-05-09T04:20:22.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:22 smithi078 ceph-mon[83176]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 98 op/s 2024-05-09T04:20:22.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:22 smithi078 ceph-mon[83176]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-09T04:20:22.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:22 smithi078 ceph-mon[83176]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-09T04:20:22.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:22 smithi078 ceph-mon[83176]: osdmap e149: 8 total, 7 up, 8 in 2024-05-09T04:20:22.864 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:22 smithi132 podman[90457]: 2024-05-09 04:20:22.490000232 +0000 UTC m=+0.018756177 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:20:22.864 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:22 smithi132 podman[90457]: 2024-05-09 04:20:22.612655705 +0000 UTC m=+0.141411677 container create d280e4ede3eb8284c409babf867903e21dbeed170b994d7c04f9833766f88190 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240507, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:20:23.163 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:23 smithi132 ceph-mon[66347]: osdmap e150: 8 total, 7 up, 8 in 2024-05-09T04:20:23.163 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:22 smithi132 podman[90457]: 2024-05-09 04:20:22.865254627 +0000 UTC m=+0.394010567 container init d280e4ede3eb8284c409babf867903e21dbeed170b994d7c04f9833766f88190 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-05-09T04:20:23.163 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:22 smithi132 podman[90457]: 2024-05-09 04:20:22.868702178 +0000 UTC m=+0.397458113 container start d280e4ede3eb8284c409babf867903e21dbeed170b994d7c04f9833766f88190 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-05-09T04:20:23.163 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:22 smithi132 podman[90457]: 2024-05-09 04:20:22.948554188 +0000 UTC m=+0.477310127 container attach d280e4ede3eb8284c409babf867903e21dbeed170b994d7c04f9833766f88190 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:20:23.459 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:23 smithi132 podman[90457]: 2024-05-09 04:20:23.164382099 +0000 UTC m=+0.693138037 container died d280e4ede3eb8284c409babf867903e21dbeed170b994d7c04f9833766f88190 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504) 2024-05-09T04:20:23.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:23 smithi078 ceph-mon[83176]: osdmap e150: 8 total, 7 up, 8 in 2024-05-09T04:20:23.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:23 smithi078 ceph-mon[88406]: osdmap e150: 8 total, 7 up, 8 in 2024-05-09T04:20:24.076 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:23 smithi132 podman[90483]: 2024-05-09 04:20:23.760908291 +0000 UTC m=+0.590783770 container remove d280e4ede3eb8284c409babf867903e21dbeed170b994d7c04f9833766f88190 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-deactivate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True) 2024-05-09T04:20:24.076 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:23 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.7.service: Deactivated successfully. 2024-05-09T04:20:24.076 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:23 smithi132 systemd[1]: Stopped Ceph osd.7 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:20:24.076 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:23 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.7.service: Consumed 17.895s CPU time. 2024-05-09T04:20:24.076 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:23 smithi132 systemd[1]: Starting Ceph osd.7 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:20:24.337 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:24 smithi132 ceph-mon[66347]: pgmap v240: 161 pgs: 1 active+undersized+degraded, 20 stale+active+clean, 6 active+undersized, 134 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 0 B/s wr, 19 op/s; 1/723 objects degraded (0.138%) 2024-05-09T04:20:24.337 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:24 smithi132 ceph-mon[66347]: Health check failed: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded (PG_DEGRADED) 2024-05-09T04:20:24.338 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 podman[90587]: 2024-05-09 04:20:24.101359592 +0000 UTC m=+0.018006002 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:20:24.338 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 podman[90587]: 2024-05-09 04:20:24.211982827 +0000 UTC m=+0.128629214 container create addd3cca62dd48f49eda53493e7acf2011e1fc456ac35868d83a560fee9eca6e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:20:24.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:24 smithi078 ceph-mon[88406]: pgmap v240: 161 pgs: 1 active+undersized+degraded, 20 stale+active+clean, 6 active+undersized, 134 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 0 B/s wr, 19 op/s; 1/723 objects degraded (0.138%) 2024-05-09T04:20:24.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:24 smithi078 ceph-mon[88406]: Health check failed: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded (PG_DEGRADED) 2024-05-09T04:20:24.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:24 smithi078 ceph-mon[83176]: pgmap v240: 161 pgs: 1 active+undersized+degraded, 20 stale+active+clean, 6 active+undersized, 134 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 0 B/s wr, 19 op/s; 1/723 objects degraded (0.138%) 2024-05-09T04:20:24.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:24 smithi078 ceph-mon[83176]: Health check failed: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded (PG_DEGRADED) 2024-05-09T04:20:24.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 podman[90587]: 2024-05-09 04:20:24.439477514 +0000 UTC m=+0.356123892 container init addd3cca62dd48f49eda53493e7acf2011e1fc456ac35868d83a560fee9eca6e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:20:24.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 podman[90587]: 2024-05-09 04:20:24.442990941 +0000 UTC m=+0.359637319 container start addd3cca62dd48f49eda53493e7acf2011e1fc456ac35868d83a560fee9eca6e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-09T04:20:24.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 podman[90587]: 2024-05-09 04:20:24.522814393 +0000 UTC m=+0.439460789 container attach addd3cca62dd48f49eda53493e7acf2011e1fc456ac35868d83a560fee9eca6e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-05-09T04:20:25.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:25 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:25.253 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate[90601]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate[90601]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 bash[90587]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 bash[90587]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 bash[90587]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 bash[90587]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 bash[90587]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 bash[90587]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 bash[90587]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 bash[90587]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 bash[90587]: --> ceph-volume raw activate successful for osd ID: 7 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate[90601]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate[90601]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-09T04:20:25.254 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate[90601]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-09T04:20:25.255 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate[90601]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-09T04:20:25.255 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate[90601]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-05-09T04:20:25.255 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate[90601]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-09T04:20:25.255 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate[90601]: --> ceph-volume raw activate successful for osd ID: 7 2024-05-09T04:20:25.255 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:24 smithi132 podman[90587]: 2024-05-09 04:20:24.94916699 +0000 UTC m=+0.865813382 container died addd3cca62dd48f49eda53493e7acf2011e1fc456ac35868d83a560fee9eca6e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:20:25.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:25 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:25.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:25 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:25.924 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:25 smithi132 podman[90734]: 2024-05-09 04:20:25.585383919 +0000 UTC m=+0.630645514 container remove addd3cca62dd48f49eda53493e7acf2011e1fc456ac35868d83a560fee9eca6e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-activate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-05-09T04:20:25.924 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:25 smithi132 podman[90800]: 2024-05-09 04:20:25.706773075 +0000 UTC m=+0.018223066 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:20:25.924 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:25 smithi132 podman[90800]: 2024-05-09 04:20:25.82000215 +0000 UTC m=+0.131452187 container create f50fb4f7e63c87b77ddfb08143a8f69a0ff8f3de39556a1bdca8bc71076c47a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS) 2024-05-09T04:20:26.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:26 smithi132 ceph-mon[66347]: pgmap v241: 161 pgs: 13 active+undersized+degraded, 12 stale+active+clean, 18 active+undersized, 118 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 63/723 objects degraded (8.714%) 2024-05-09T04:20:26.253 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:26 smithi132 podman[90800]: 2024-05-09 04:20:26.022579791 +0000 UTC m=+0.334029789 container init f50fb4f7e63c87b77ddfb08143a8f69a0ff8f3de39556a1bdca8bc71076c47a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:20:26.253 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:26 smithi132 podman[90800]: 2024-05-09 04:20:26.025831165 +0000 UTC m=+0.337281159 container start f50fb4f7e63c87b77ddfb08143a8f69a0ff8f3de39556a1bdca8bc71076c47a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:20:26.253 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:26 smithi132 ceph-osd[90819]: -- 172.21.15.132:0/1339650883 <== mon.2 v2:172.21.15.132:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55aa0403d680 con 0x55aa04007000 2024-05-09T04:20:26.253 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:26 smithi132 bash[90800]: f50fb4f7e63c87b77ddfb08143a8f69a0ff8f3de39556a1bdca8bc71076c47a7 2024-05-09T04:20:26.253 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:26 smithi132 systemd[1]: Started Ceph osd.7 for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:20:26.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:26 smithi078 ceph-mon[83176]: pgmap v241: 161 pgs: 13 active+undersized+degraded, 12 stale+active+clean, 18 active+undersized, 118 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 63/723 objects degraded (8.714%) 2024-05-09T04:20:26.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:26 smithi078 ceph-mon[88406]: pgmap v241: 161 pgs: 13 active+undersized+degraded, 12 stale+active+clean, 18 active+undersized, 118 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 63/723 objects degraded (8.714%) 2024-05-09T04:20:27.002 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:26 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:20:26.581+0000 7f92b32217c0 -1 Falling back to public interface 2024-05-09T04:20:27.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:27.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:27.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:27.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:27.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:27.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:27.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:27.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:27.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:28.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:28 smithi132 ceph-mon[66347]: pgmap v242: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-05-09T04:20:28.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:28.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:20:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:28 smithi078 ceph-mon[83176]: pgmap v242: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-05-09T04:20:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:28.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:20:28.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:28 smithi078 ceph-mon[88406]: pgmap v242: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-05-09T04:20:28.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:28.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:20:29.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:20:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:20:29] "GET /metrics HTTP/1.1" 200 38455 "" "Prometheus/2.43.0" 2024-05-09T04:20:29.593 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:29 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:20:29.292+0000 7f92b32217c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-05-09T04:20:29.593 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:29 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:20:29.323+0000 7f92b32217c0 -1 osd.7 148 log_to_monitors true 2024-05-09T04:20:30.165 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:30 smithi132 ceph-mon[66347]: pgmap v243: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-05-09T04:20:30.166 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:30 smithi132 ceph-mon[66347]: from='osd.7 [v2:172.21.15.132:6824/3745202747,v1:172.21.15.132:6825/3745202747]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-09T04:20:30.166 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:30 smithi132 ceph-mon[66347]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-09T04:20:30.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:30 smithi078 ceph-mon[83176]: pgmap v243: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-05-09T04:20:30.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:30 smithi078 ceph-mon[83176]: from='osd.7 [v2:172.21.15.132:6824/3745202747,v1:172.21.15.132:6825/3745202747]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-09T04:20:30.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:30 smithi078 ceph-mon[83176]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-09T04:20:30.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:30 smithi078 ceph-mon[88406]: pgmap v243: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-05-09T04:20:30.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:30 smithi078 ceph-mon[88406]: from='osd.7 [v2:172.21.15.132:6824/3745202747,v1:172.21.15.132:6825/3745202747]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-09T04:20:30.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:30 smithi078 ceph-mon[88406]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-09T04:20:30.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:20:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:20:30.621+0000 7f92a0670700 -1 osd.7 148 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-09T04:20:31.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:31 smithi132 ceph-mon[66347]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-09T04:20:31.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:31 smithi132 ceph-mon[66347]: from='osd.7 [v2:172.21.15.132:6824/3745202747,v1:172.21.15.132:6825/3745202747]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:20:31.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:31 smithi132 ceph-mon[66347]: osdmap e151: 8 total, 7 up, 8 in 2024-05-09T04:20:31.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:31 smithi132 ceph-mon[66347]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:20:31.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:31 smithi132 ceph-mon[66347]: from='osd.7 ' entity='osd.7' 2024-05-09T04:20:31.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:31 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:31.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:31 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:31.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[83176]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-09T04:20:31.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[83176]: from='osd.7 [v2:172.21.15.132:6824/3745202747,v1:172.21.15.132:6825/3745202747]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:20:31.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[83176]: osdmap e151: 8 total, 7 up, 8 in 2024-05-09T04:20:31.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[83176]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:20:31.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[83176]: from='osd.7 ' entity='osd.7' 2024-05-09T04:20:31.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:31.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[88406]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-09T04:20:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[88406]: from='osd.7 [v2:172.21.15.132:6824/3745202747,v1:172.21.15.132:6825/3745202747]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:20:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[88406]: osdmap e151: 8 total, 7 up, 8 in 2024-05-09T04:20:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[88406]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi132", "root=default"]}]: dispatch 2024-05-09T04:20:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[88406]: from='osd.7 ' entity='osd.7' 2024-05-09T04:20:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:31.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:31 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:32.417 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:32 smithi132 ceph-mon[66347]: pgmap v245: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-05-09T04:20:32.417 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:32 smithi132 ceph-mon[66347]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:20:32.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:32 smithi132 ceph-mon[66347]: osd.7 [v2:172.21.15.132:6824/3745202747,v1:172.21.15.132:6825/3745202747] boot 2024-05-09T04:20:32.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:32 smithi132 ceph-mon[66347]: osdmap e152: 8 total, 8 up, 8 in 2024-05-09T04:20:32.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:20:32.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:32.418 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:32.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[88406]: pgmap v245: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-05-09T04:20:32.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[88406]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:20:32.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[88406]: osd.7 [v2:172.21.15.132:6824/3745202747,v1:172.21.15.132:6825/3745202747] boot 2024-05-09T04:20:32.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[88406]: osdmap e152: 8 total, 8 up, 8 in 2024-05-09T04:20:32.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:20:32.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:32.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:32.475 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[83176]: pgmap v245: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-05-09T04:20:32.475 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[83176]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-09T04:20:32.476 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[83176]: osd.7 [v2:172.21.15.132:6824/3745202747,v1:172.21.15.132:6825/3745202747] boot 2024-05-09T04:20:32.476 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[83176]: osdmap e152: 8 total, 8 up, 8 in 2024-05-09T04:20:32.476 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-09T04:20:32.476 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:32.476 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:32.476 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:20:33.163 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:20:33.163 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (9m) 2m ago 31m 28.0M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:20:33.163 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (8m) 2s ago 30m 80.0M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:20:33.163 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (10m) 2m ago 30m 67.2M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:20:33.163 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (10m) 2s ago 35m 468M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:20:33.163 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (20m) 2m ago 37m 524M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (7m) 2m ago 37m 51.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (7m) 2s ago 36m 48.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (6m) 2m ago 36m 39.8M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (9m) 2m ago 31m 16.6M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (9m) 2s ago 31m 17.1M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 2m ago 35m 48.0M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (4m) 2m ago 35m 46.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 34m 40.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (2m) 2m ago 34m 12.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a cfe832ae10c3 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (113s) 2s ago 33m 48.0M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a cdf9ea05dcb0 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (77s) 2s ago 33m 52.8M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 408a908ef3cc 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (42s) 2s ago 33m 44.2M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 01f14aea01f3 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (7s) 2s ago 32m 12.6M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a f50fb4f7e63c 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (9m) 2s ago 32m 72.2M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:20:33.164 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (30m) 2m ago 30m 80.2M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:20:33.165 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (30m) 2s ago 30m 82.7M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:20:33.570 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 8 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:20:33.571 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-09T04:20:33.572 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 13 2024-05-09T04:20:33.572 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:20:33.572 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:20:33.650 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:33 smithi132 ceph-mon[66347]: from='client.54547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:33.650 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:33 smithi132 ceph-mon[66347]: osdmap e153: 8 total, 8 up, 8 in 2024-05-09T04:20:33.650 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:33 smithi132 ceph-mon[66347]: Health check update: Degraded data redundancy: 96/723 objects degraded (13.278%), 26 pgs degraded (PG_DEGRADED) 2024-05-09T04:20:33.650 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:33 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/1099650779' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:33.916 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:20:33.917 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:20:33.917 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:20:33.917 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-09T04:20:33.917 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-05-09T04:20:33.917 INFO:teuthology.orchestra.run.smithi078.stdout: "osd" 2024-05-09T04:20:33.917 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-05-09T04:20:33.917 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "8/8 daemons upgraded", 2024-05-09T04:20:33.917 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-05-09T04:20:33.917 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:20:33.917 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:20:33.918 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:33 smithi078 ceph-mon[83176]: from='client.54547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:33.918 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:33 smithi078 ceph-mon[83176]: osdmap e153: 8 total, 8 up, 8 in 2024-05-09T04:20:33.918 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:33 smithi078 ceph-mon[83176]: Health check update: Degraded data redundancy: 96/723 objects degraded (13.278%), 26 pgs degraded (PG_DEGRADED) 2024-05-09T04:20:33.918 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:33 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/1099650779' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:33.918 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:33 smithi078 ceph-mon[88406]: from='client.54547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:33.918 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:33 smithi078 ceph-mon[88406]: osdmap e153: 8 total, 8 up, 8 in 2024-05-09T04:20:33.918 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:33 smithi078 ceph-mon[88406]: Health check update: Degraded data redundancy: 96/723 objects degraded (13.278%), 26 pgs degraded (PG_DEGRADED) 2024-05-09T04:20:33.918 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:33 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/1099650779' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:34.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:34 smithi132 ceph-mon[66347]: from='client.54553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:34.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:34 smithi132 ceph-mon[66347]: pgmap v248: 161 pgs: 25 active+undersized+degraded, 40 active+undersized, 96 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 95/723 objects degraded (13.140%) 2024-05-09T04:20:34.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:34 smithi132 ceph-mon[66347]: from='client.54559 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:34 smithi078 ceph-mon[83176]: from='client.54553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:34 smithi078 ceph-mon[83176]: pgmap v248: 161 pgs: 25 active+undersized+degraded, 40 active+undersized, 96 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 95/723 objects degraded (13.140%) 2024-05-09T04:20:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:34 smithi078 ceph-mon[83176]: from='client.54559 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:35.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:34 smithi078 ceph-mon[88406]: from='client.54553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:35.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:34 smithi078 ceph-mon[88406]: pgmap v248: 161 pgs: 25 active+undersized+degraded, 40 active+undersized, 96 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 95/723 objects degraded (13.140%) 2024-05-09T04:20:35.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:34 smithi078 ceph-mon[88406]: from='client.54559 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:36.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:36.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-09T04:20:36.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-09T04:20:36.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-09T04:20:36.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:36.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:36.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:36.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:36.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:36.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-09T04:20:36.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-09T04:20:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-09T04:20:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-09T04:20:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-09T04:20:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-09T04:20:36.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-09T04:20:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:20:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:20:36.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-09T04:20:36.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-09T04:20:36.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-09T04:20:36.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-09T04:20:36.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-09T04:20:36.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-09T04:20:36.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-09T04:20:36.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-09T04:20:36.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-09T04:20:36.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-09T04:20:36.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-09T04:20:36.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-09T04:20:36.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-09T04:20:36.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-09T04:20:36.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-09T04:20:37.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: pgmap v249: 161 pgs: 18 active+undersized+degraded, 31 active+undersized, 112 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 62/723 objects degraded (8.575%) 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all osd 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: Upgrade: Setting require_osd_release to 19 squid 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: osdmap e154: 8 total, 8 up, 8 in 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:20:37.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:20:37.006 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:20:37.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:20:37.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:20:37.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:37.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:37.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:20:37.007 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:36 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: pgmap v249: 161 pgs: 18 active+undersized+degraded, 31 active+undersized, 112 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 62/723 objects degraded (8.575%) 2024-05-09T04:20:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all osd 2024-05-09T04:20:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: Upgrade: Setting require_osd_release to 19 squid 2024-05-09T04:20:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-09T04:20:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-09T04:20:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: osdmap e154: 8 total, 8 up, 8 in 2024-05-09T04:20:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:20:37.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: pgmap v249: 161 pgs: 18 active+undersized+degraded, 31 active+undersized, 112 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 62/723 objects degraded (8.575%) 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all osd 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: Upgrade: Setting require_osd_release to 19 squid 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: osdmap e154: 8 total, 8 up, 8 in 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:20:37.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:20:37.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:20:37.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:20:37.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:20:37.017 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:20:37.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:20:37.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-09T04:20:37.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:20:37.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:20:37.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:20:37.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:20:37.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:20:37.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:20:37.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:20:37.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:20:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:20:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:20:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:20:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:20:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.019 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:20:37.019 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:20:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:20:37.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:20:37.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:20:37.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:20:37.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:37.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:37.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:20:37.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:37.022 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:20:37.022 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:20:37.022 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:37.022 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:37.022 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:20:37.022 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:36 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:38.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:37 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mds 2024-05-09T04:20:38.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:37 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:20:38.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:37 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:20:38.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:37 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all nfs 2024-05-09T04:20:38.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:37 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:20:38.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:37 smithi132 ceph-mon[66347]: Upgrade: Finalizing container_image settings 2024-05-09T04:20:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:37 smithi132 ceph-mon[66347]: Upgrade: Complete! 2024-05-09T04:20:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:37 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:37 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:37 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:20:38.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:37 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mds 2024-05-09T04:20:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:20:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:20:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all nfs 2024-05-09T04:20:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:20:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[83176]: Upgrade: Finalizing container_image settings 2024-05-09T04:20:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[83176]: Upgrade: Complete! 2024-05-09T04:20:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:20:38.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mds 2024-05-09T04:20:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:20:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:20:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all nfs 2024-05-09T04:20:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:20:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[88406]: Upgrade: Finalizing container_image settings 2024-05-09T04:20:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[88406]: Upgrade: Complete! 2024-05-09T04:20:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:20:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:20:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:20:38.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:37 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:39.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:38 smithi132 ceph-mon[66347]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:20:39.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:38 smithi132 ceph-mon[66347]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/723 objects degraded (8.575%), 18 pgs degraded) 2024-05-09T04:20:39.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:38 smithi132 ceph-mon[66347]: Cluster is now healthy 2024-05-09T04:20:39.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:38 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:39.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:38 smithi078 ceph-mon[83176]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:20:39.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:38 smithi078 ceph-mon[83176]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/723 objects degraded (8.575%), 18 pgs degraded) 2024-05-09T04:20:39.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:38 smithi078 ceph-mon[83176]: Cluster is now healthy 2024-05-09T04:20:39.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:38 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:39.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:38 smithi078 ceph-mon[88406]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:20:39.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:38 smithi078 ceph-mon[88406]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/723 objects degraded (8.575%), 18 pgs degraded) 2024-05-09T04:20:39.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:38 smithi078 ceph-mon[88406]: Cluster is now healthy 2024-05-09T04:20:39.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:38 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:39.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:20:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:20:39] "GET /metrics HTTP/1.1" 200 38455 "" "Prometheus/2.43.0" 2024-05-09T04:20:41.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:40 smithi132 ceph-mon[66347]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-09T04:20:41.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:40 smithi078 ceph-mon[83176]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-09T04:20:41.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:40 smithi078 ceph-mon[88406]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-09T04:20:43.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:42 smithi132 ceph-mon[66347]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:20:43.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:42 smithi078 ceph-mon[83176]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:20:43.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:42 smithi078 ceph-mon[88406]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:20:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:20:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:43 smithi078 ceph-mon[88406]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:20:44.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:20:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:43 smithi078 ceph-mon[83176]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:20:44.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:20:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:43 smithi132 ceph-mon[66347]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:20:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:20:45.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:44 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:45.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:44 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:45.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:44 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:46.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:45 smithi132 ceph-mon[66347]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:20:46.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:45 smithi078 ceph-mon[83176]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:20:46.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:45 smithi078 ceph-mon[88406]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-09T04:20:48.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:48 smithi132 ceph-mon[66347]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 965 B/s rd, 0 op/s 2024-05-09T04:20:48.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:48 smithi078 ceph-mon[83176]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 965 B/s rd, 0 op/s 2024-05-09T04:20:48.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:48 smithi078 ceph-mon[88406]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 965 B/s rd, 0 op/s 2024-05-09T04:20:49.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:20:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:20:49] "GET /metrics HTTP/1.1" 200 38454 "" "Prometheus/2.43.0" 2024-05-09T04:20:50.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:50 smithi132 ceph-mon[66347]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:20:50.510 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:50 smithi078 ceph-mon[83176]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:20:50.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:50 smithi078 ceph-mon[88406]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:20:52.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:52 smithi132 ceph-mon[66347]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:20:52.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:52 smithi078 ceph-mon[88406]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:20:52.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:52 smithi078 ceph-mon[83176]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:20:54.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:54 smithi132 ceph-mon[66347]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:20:54.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:54 smithi078 ceph-mon[83176]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:20:54.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:54 smithi078 ceph-mon[88406]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:20:55.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:55 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:55.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:55 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:55.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:55 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:20:56.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:56 smithi132 ceph-mon[66347]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:20:56.510 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:56 smithi078 ceph-mon[83176]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:20:56.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:56 smithi078 ceph-mon[88406]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:20:58.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:58 smithi132 ceph-mon[66347]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:20:58.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:20:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:20:58.510 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:58 smithi078 ceph-mon[83176]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:20:58.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:20:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:20:58.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:58 smithi078 ceph-mon[88406]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:20:58.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:20:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:20:59.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:20:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:20:59] "GET /metrics HTTP/1.1" 200 38456 "" "Prometheus/2.43.0" 2024-05-09T04:21:00.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:00 smithi132 ceph-mon[66347]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:00.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:00 smithi078 ceph-mon[88406]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:00.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:00 smithi078 ceph-mon[83176]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:02.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:02 smithi132 ceph-mon[66347]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:02.510 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:02 smithi078 ceph-mon[83176]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:02.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:02 smithi078 ceph-mon[88406]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:04.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:04 smithi132 ceph-mon[66347]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:04.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:04 smithi078 ceph-mon[83176]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:04.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:04 smithi078 ceph-mon[88406]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:05.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T04:21:05.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:05 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:05.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:05 smithi078 ceph-mon[83176]: from='client.54577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:05.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:05 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:05.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:05 smithi078 ceph-mon[88406]: from='client.54577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:05.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:05 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:05.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:05 smithi132 ceph-mon[66347]: from='client.54577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:06.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:06 smithi078 ceph-mon[83176]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:06.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:06 smithi078 ceph-mon[88406]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:06.404 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:06 smithi132 ceph-mon[66347]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:07.016 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:21:07.016 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (9m) 2m ago 31m 28.0M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:21:07.016 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (9m) 36s ago 31m 80.0M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:21:07.016 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (10m) 2m ago 30m 67.2M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:21:07.016 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (10m) 36s ago 36m 468M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:21:07.016 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (20m) 2m ago 38m 524M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:21:07.016 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (7m) 2m ago 38m 51.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:21:07.016 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (8m) 36s ago 36m 48.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (7m) 2m ago 36m 39.8M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (10m) 2m ago 32m 16.6M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (10m) 36s ago 32m 17.1M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 2m ago 36m 48.0M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (4m) 2m ago 35m 46.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 35m 40.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (3m) 2m ago 34m 12.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a cfe832ae10c3 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (2m) 36s ago 34m 48.0M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a cdf9ea05dcb0 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (111s) 36s ago 34m 52.8M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 408a908ef3cc 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (76s) 36s ago 33m 44.2M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 01f14aea01f3 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (41s) 36s ago 33m 12.6M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a f50fb4f7e63c 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (10m) 36s ago 32m 72.2M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (31m) 2m ago 30m 80.2M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:21:07.017 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (30m) 36s ago 30m 82.7M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:21:07.756 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-05-09T04:21:08.377 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:08 smithi078 ceph-mon[83176]: from='client.54583 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:08.377 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:08 smithi078 ceph-mon[83176]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:08.377 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:08 smithi078 ceph-mon[88406]: from='client.54583 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:08.377 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:08 smithi078 ceph-mon[88406]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:08.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:08 smithi132 ceph-mon[66347]: from='client.54583 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:08.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:08 smithi132 ceph-mon[66347]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:09.367 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:21:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:21:09] "GET /metrics HTTP/1.1" 200 38456 "" "Prometheus/2.43.0" 2024-05-09T04:21:09.888 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:21:10.230 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:10 smithi078 ceph-mon[88406]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:10.230 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:10 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/1675666831' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:10.230 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:10 smithi078 ceph-mon[83176]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:10.230 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:10 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/1675666831' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:10.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:10 smithi132 ceph-mon[66347]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:10.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:10 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/1675666831' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:10.571 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-05-09T04:21:12.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:12 smithi078 ceph-mon[88406]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:12.215 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:12 smithi078 ceph-mon[83176]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:12.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:12 smithi132 ceph-mon[66347]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:12.778 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)" 2024-05-09T04:21:13.518 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade status' 2024-05-09T04:21:13.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:13 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/3497526137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:13.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:21:13.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:13 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/3497526137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:13.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:21:14.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:13 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/3497526137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:14.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:21:14.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:14 smithi078 ceph-mon[83176]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:14.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:14 smithi078 ceph-mon[88406]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:15.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:14 smithi132 ceph-mon[66347]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:15.496 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:21:15.496 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": null, 2024-05-09T04:21:15.496 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": false, 2024-05-09T04:21:15.496 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "", 2024-05-09T04:21:15.496 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:21:15.496 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": null, 2024-05-09T04:21:15.496 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-05-09T04:21:15.496 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:21:15.496 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:21:15.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:15 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:15.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:15 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:16.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:15 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:16.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph health detail' 2024-05-09T04:21:16.864 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:16 smithi078 ceph-mon[88406]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:16.864 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:16 smithi078 ceph-mon[88406]: from='client.54601 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:16.864 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:16 smithi078 ceph-mon[83176]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:16.864 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:16 smithi078 ceph-mon[83176]: from='client.54601 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:16 smithi132 ceph-mon[66347]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:16 smithi132 ceph-mon[66347]: from='client.54601 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:18.241 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-05-09T04:21:18.882 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:18 smithi078 ceph-mon[83176]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:18.882 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:18 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/106991669' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:21:18.882 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:18 smithi078 ceph-mon[88406]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:18.882 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:18 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/106991669' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:21:18.906 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-05-09T04:21:19.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:18 smithi132 ceph-mon[66347]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:19.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:18 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/106991669' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:21:19.213 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:21:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:21:19] "GET /metrics HTTP/1.1" 200 38455 "" "Prometheus/2.43.0" 2024-05-09T04:21:21.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:20 smithi132 ceph-mon[66347]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:21.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:20 smithi078 ceph-mon[83176]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:20 smithi078 ceph-mon[88406]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:22.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:21 smithi132 ceph-mon[66347]: from='client.44673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:22.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:21 smithi078 ceph-mon[83176]: from='client.44673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:22.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:21 smithi078 ceph-mon[88406]: from='client.44673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:22.944 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:21:23.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:22 smithi078 ceph-mon[83176]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:23.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:22 smithi078 ceph-mon[88406]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:23.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:22 smithi132 ceph-mon[66347]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:23.728 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-09T04:21:24.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[83176]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:21:24.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:24.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:24.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:24.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:21:24.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:24.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[83176]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:24.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[83176]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:21:24.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[88406]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:21:24.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:24.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:24.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:24.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:21:24.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:24.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[88406]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:24.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:23 smithi078 ceph-mon[88406]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:21:24.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:23 smithi132 ceph-mon[66347]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:21:24.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:23 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:24.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:23 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:24.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:23 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:24.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:23 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:21:24.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:23 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:24.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:23 smithi132 ceph-mon[66347]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:24.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:23 smithi132 ceph-mon[66347]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:21:25.144 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:24 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:25.145 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:24 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:25.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:24 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:25.998 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:25 smithi078 ceph-mon[83176]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:25.998 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:25 smithi078 ceph-mon[88406]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:26.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:25 smithi132 ceph-mon[66347]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:27.033 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:21:27.737 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:21:27.737 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (10m) 3m ago 32m 28.0M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:21:27.737 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (9m) 57s ago 31m 80.0M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:21:27.737 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (11m) 3m ago 31m 67.2M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:21:27.737 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (11m) 57s ago 36m 468M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:21:27.737 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (21m) 3m ago 38m 524M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:21:27.737 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (7m) 3m ago 38m 51.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:21:27.737 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (8m) 57s ago 36m 48.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:21:27.737 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (7m) 3m ago 37m 39.8M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:21:27.737 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (10m) 3m ago 32m 16.6M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:21:27.737 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (10m) 57s ago 32m 17.1M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:21:27.738 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 3m ago 36m 48.0M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:21:27.738 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 3m ago 35m 46.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:21:27.738 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 3m ago 35m 40.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:21:27.738 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (3m) 3m ago 35m 12.7M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a cfe832ae10c3 2024-05-09T04:21:27.738 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (2m) 57s ago 34m 48.0M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a cdf9ea05dcb0 2024-05-09T04:21:27.738 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (2m) 57s ago 34m 52.8M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 408a908ef3cc 2024-05-09T04:21:27.738 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (96s) 57s ago 34m 44.2M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 01f14aea01f3 2024-05-09T04:21:27.738 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (62s) 57s ago 33m 12.6M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a f50fb4f7e63c 2024-05-09T04:21:27.738 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (10m) 57s ago 32m 72.2M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:21:27.738 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (31m) 3m ago 31m 80.2M - 17.2.0 e1d6a67b021e 9a01044ca5d0 2024-05-09T04:21:27.738 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (31m) 57s ago 31m 82.7M - 17.2.0 e1d6a67b021e 5d9dd78675e9 2024-05-09T04:21:28.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mgr 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mon 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all crash 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all osd 2024-05-09T04:21:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mds 2024-05-09T04:21:28.005 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:27 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:21:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:21:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mgr 2024-05-09T04:21:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mon 2024-05-09T04:21:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all crash 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all osd 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mds 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mgr 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mon 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all crash 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all osd 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mds 2024-05-09T04:21:28.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:27 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.146 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:21:28.146 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:21:28.146 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:21:28.146 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:21:28.146 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:21:28.146 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:21:28.146 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:21:28.146 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:21:28.146 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 8 2024-05-09T04:21:28.147 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:21:28.147 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:21:28.147 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-09T04:21:28.147 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:21:28.147 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:21:28.147 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-09T04:21:28.147 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 13 2024-05-09T04:21:28.147 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:21:28.147 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:21:28.500 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:21:28.500 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:21:28.500 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:21:28.500 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-05-09T04:21:28.500 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:21:28.500 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "0/2 daemons upgraded", 2024-05-09T04:21:28.500 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading rgw daemons", 2024-05-09T04:21:28.500 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:21:28.500 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:21:28.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[83176]: from='client.54619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:28.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[83176]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:28.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[83176]: from='client.54625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:28.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[83176]: from='client.54631 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:28.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:21:28.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/1431426618' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T04:21:28.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[88406]: from='client.54619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[88406]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[88406]: from='client.54625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[88406]: from='client.54631 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:21:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/1431426618' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T04:21:28.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:28 smithi132 ceph-mon[66347]: from='client.54619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:28 smithi132 ceph-mon[66347]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:28 smithi132 ceph-mon[66347]: from='client.54625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:28 smithi132 ceph-mon[66347]: from='client.54631 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:21:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:28 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/1431426618' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:29.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi078.bfidtv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T04:21:29.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:29.179 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:21:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:21:29] "GET /metrics HTTP/1.1" 200 38453 "" "Prometheus/2.43.0" 2024-05-09T04:21:29.908 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:29 smithi078 ceph-mon[83176]: Upgrade: Updating rgw.foo.smithi078.bfidtv (1/2) 2024-05-09T04:21:29.909 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:29 smithi078 ceph-mon[83176]: Deploying daemon rgw.foo.smithi078.bfidtv on smithi078 2024-05-09T04:21:29.909 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:29 smithi078 ceph-mon[83176]: from='client.54643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:29.909 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:29 smithi078 ceph-mon[88406]: Upgrade: Updating rgw.foo.smithi078.bfidtv (1/2) 2024-05-09T04:21:29.909 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:29 smithi078 ceph-mon[88406]: Deploying daemon rgw.foo.smithi078.bfidtv on smithi078 2024-05-09T04:21:29.909 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:29 smithi078 ceph-mon[88406]: from='client.54643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:30.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:29 smithi132 ceph-mon[66347]: Upgrade: Updating rgw.foo.smithi078.bfidtv (1/2) 2024-05-09T04:21:30.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:29 smithi132 ceph-mon[66347]: Deploying daemon rgw.foo.smithi078.bfidtv on smithi078 2024-05-09T04:21:30.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:29 smithi132 ceph-mon[66347]: from='client.54643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:21:30.947 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:30 smithi078 ceph-mon[88406]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:30.947 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:30 smithi078 ceph-mon[83176]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:31.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:30 smithi132 ceph-mon[66347]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:21:33.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:32 smithi132 ceph-mon[66347]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:33.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:33.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:32 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:33.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:32 smithi078 ceph-mon[83176]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:33.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:33.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:32 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:33.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:32 smithi078 ceph-mon[88406]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:33.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:33.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:32 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:35.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:34 smithi132 ceph-mon[66347]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:35.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:35.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T04:21:35.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:34 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:34 smithi078 ceph-mon[83176]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T04:21:35.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:34 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:35.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:34 smithi078 ceph-mon[88406]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:21:35.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:35.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi132.pcutwm", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-09T04:21:35.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:34 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:35.737 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:35 smithi132 ceph-mon[66347]: Upgrade: Updating rgw.foo.smithi132.pcutwm (2/2) 2024-05-09T04:21:35.737 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:35 smithi132 ceph-mon[66347]: Deploying daemon rgw.foo.smithi132.pcutwm on smithi132 2024-05-09T04:21:35.737 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:35 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:36.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:35 smithi078 ceph-mon[83176]: Upgrade: Updating rgw.foo.smithi132.pcutwm (2/2) 2024-05-09T04:21:36.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:35 smithi078 ceph-mon[83176]: Deploying daemon rgw.foo.smithi132.pcutwm on smithi132 2024-05-09T04:21:36.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:35 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:36.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:35 smithi078 ceph-mon[88406]: Upgrade: Updating rgw.foo.smithi132.pcutwm (2/2) 2024-05-09T04:21:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:35 smithi078 ceph-mon[88406]: Deploying daemon rgw.foo.smithi132.pcutwm on smithi132 2024-05-09T04:21:36.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:35 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:36.868 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:36 smithi132 ceph-mon[66347]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 43 op/s 2024-05-09T04:21:37.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:36 smithi078 ceph-mon[83176]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 43 op/s 2024-05-09T04:21:37.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:36 smithi078 ceph-mon[88406]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 43 op/s 2024-05-09T04:21:39.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:38 smithi132 ceph-mon[66347]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 85 B/s wr, 124 op/s 2024-05-09T04:21:39.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:38 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:39.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:38 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:39.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:38 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:39.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:38 smithi078 ceph-mon[83176]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 85 B/s wr, 124 op/s 2024-05-09T04:21:39.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:38 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:39.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:38 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:39.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:38 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:39.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:38 smithi078 ceph-mon[88406]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 85 B/s wr, 124 op/s 2024-05-09T04:21:39.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:38 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:39.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:38 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:39.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:38 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:39.395 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:21:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:21:39] "GET /metrics HTTP/1.1" 200 38453 "" "Prometheus/2.43.0" 2024-05-09T04:21:41.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:40 smithi132 ceph-mon[66347]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 85 B/s wr, 167 op/s 2024-05-09T04:21:41.010 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:40 smithi078 ceph-mon[88406]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 85 B/s wr, 167 op/s 2024-05-09T04:21:41.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:40 smithi078 ceph-mon[83176]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 85 B/s wr, 167 op/s 2024-05-09T04:21:43.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:42 smithi132 ceph-mon[66347]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 146 KiB/s rd, 170 B/s wr, 230 op/s 2024-05-09T04:21:43.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:43.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:43.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:43.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:42 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:43.010 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:42 smithi078 ceph-mon[88406]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 146 KiB/s rd, 170 B/s wr, 230 op/s 2024-05-09T04:21:43.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:42 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:43.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:42 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:43.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:42 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:43.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:42 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:43.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:42 smithi078 ceph-mon[83176]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 146 KiB/s rd, 170 B/s wr, 230 op/s 2024-05-09T04:21:43.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:43.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:43.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:43.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:42 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:44.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:21:44.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:44.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:44.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:44.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:44.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:21:44.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:44.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:44.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:44.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:44.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:21:44.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:44.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:44.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:44.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:45.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:44 smithi132 ceph-mon[66347]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 247 op/s 2024-05-09T04:21:45.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:44 smithi078 ceph-mon[83176]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 247 op/s 2024-05-09T04:21:45.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:44 smithi078 ceph-mon[88406]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 247 op/s 2024-05-09T04:21:46.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:45 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:46.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:45 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:46.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:45 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:47.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:46 smithi132 ceph-mon[66347]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 248 op/s 2024-05-09T04:21:47.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:46 smithi078 ceph-mon[88406]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 248 op/s 2024-05-09T04:21:47.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:46 smithi078 ceph-mon[83176]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 248 op/s 2024-05-09T04:21:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:21:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:48.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi078.bfidtv"}]: dispatch 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi078.bfidtv"}]': finished 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi132.pcutwm"}]: dispatch 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi132.pcutwm"}]': finished 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.255 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:21:48.256 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:21:48.257 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:48.258 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:48.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:21:48.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:48.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:48.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:21:48.259 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:47 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi078.bfidtv"}]: dispatch 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi078.bfidtv"}]': finished 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi132.pcutwm"}]: dispatch 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi132.pcutwm"}]': finished 2024-05-09T04:21:48.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi078.bfidtv"}]: dispatch 2024-05-09T04:21:48.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi078.bfidtv"}]': finished 2024-05-09T04:21:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi132.pcutwm"}]: dispatch 2024-05-09T04:21:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi132.pcutwm"}]': finished 2024-05-09T04:21:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:21:48.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:21:48.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:21:48.267 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:48.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:48.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:21:48.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:21:48.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-09T04:21:48.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-09T04:21:48.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-09T04:21:48.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:48.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:48.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:21:48.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:48.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:21:48.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:21:48.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:21:48.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:47 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:49.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[88406]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 170 B/s wr, 205 op/s 2024-05-09T04:21:49.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all rgw 2024-05-09T04:21:49.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:21:49.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:21:49.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all nfs 2024-05-09T04:21:49.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:21:49.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[88406]: Upgrade: Finalizing container_image settings 2024-05-09T04:21:49.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[88406]: Upgrade: Complete! 2024-05-09T04:21:49.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[88406]: Checking dashboard <-> RGW credentials 2024-05-09T04:21:49.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:49.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[83176]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 170 B/s wr, 205 op/s 2024-05-09T04:21:49.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all rgw 2024-05-09T04:21:49.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:21:49.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:21:49.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all nfs 2024-05-09T04:21:49.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:21:49.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[83176]: Upgrade: Finalizing container_image settings 2024-05-09T04:21:49.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[83176]: Upgrade: Complete! 2024-05-09T04:21:49.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[83176]: Checking dashboard <-> RGW credentials 2024-05-09T04:21:49.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:48 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:49.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:48 smithi132 ceph-mon[66347]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 170 B/s wr, 205 op/s 2024-05-09T04:21:49.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:48 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all rgw 2024-05-09T04:21:49.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:48 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:21:49.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:48 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:21:49.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:48 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all nfs 2024-05-09T04:21:49.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:48 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all nvmeof 2024-05-09T04:21:49.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:48 smithi132 ceph-mon[66347]: Upgrade: Finalizing container_image settings 2024-05-09T04:21:49.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:48 smithi132 ceph-mon[66347]: Upgrade: Complete! 2024-05-09T04:21:49.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:48 smithi132 ceph-mon[66347]: Checking dashboard <-> RGW credentials 2024-05-09T04:21:49.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:48 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:21:49.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:21:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:21:49] "GET /metrics HTTP/1.1" 200 38487 "" "Prometheus/2.43.0" 2024-05-09T04:21:51.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:50 smithi132 ceph-mon[66347]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 85 B/s wr, 125 op/s 2024-05-09T04:21:51.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:50 smithi078 ceph-mon[83176]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 85 B/s wr, 125 op/s 2024-05-09T04:21:51.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:50 smithi078 ceph-mon[88406]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 85 B/s wr, 125 op/s 2024-05-09T04:21:52.004 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:21:51 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=cleanup t=2024-05-09T04:21:51.547315222Z level=info msg="Completed cleanup jobs" duration=3.950697ms 2024-05-09T04:21:53.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:52 smithi132 ceph-mon[66347]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 85 B/s wr, 83 op/s 2024-05-09T04:21:53.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:52 smithi078 ceph-mon[83176]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 85 B/s wr, 83 op/s 2024-05-09T04:21:53.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:52 smithi078 ceph-mon[88406]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 85 B/s wr, 83 op/s 2024-05-09T04:21:55.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:54 smithi132 ceph-mon[66347]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 19 op/s 2024-05-09T04:21:55.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:54 smithi078 ceph-mon[83176]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 19 op/s 2024-05-09T04:21:55.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:54 smithi078 ceph-mon[88406]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 19 op/s 2024-05-09T04:21:56.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:55 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:56.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:55 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:56.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:55 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:21:57.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:56 smithi132 ceph-mon[66347]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-05-09T04:21:57.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:56 smithi078 ceph-mon[83176]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-05-09T04:21:57.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:56 smithi078 ceph-mon[88406]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-05-09T04:21:59.172 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:58 smithi078 ceph-mon[88406]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-09T04:21:59.173 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:21:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:21:59.173 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:58 smithi078 ceph-mon[83176]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-09T04:21:59.173 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:21:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:21:59.173 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:21:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:21:59] "GET /metrics HTTP/1.1" 200 38485 "" "Prometheus/2.43.0" 2024-05-09T04:21:59.252 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:58 smithi132 ceph-mon[66347]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-09T04:21:59.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:21:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:21:59.559 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T04:22:00.967 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:00 smithi078 ceph-mon[88406]: from='client.44787 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:00.967 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:00 smithi078 ceph-mon[88406]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-05-09T04:22:00.967 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:00 smithi078 ceph-mon[83176]: from='client.44787 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:00.967 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:00 smithi078 ceph-mon[83176]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-05-09T04:22:01.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:00 smithi132 ceph-mon[66347]: from='client.44787 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:01.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:00 smithi132 ceph-mon[66347]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-05-09T04:22:01.668 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:22:01.668 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (10m) 19s ago 32m 27.2M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:22:01.668 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (10m) 19s ago 32m 82.0M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:22:01.668 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (11m) 19s ago 31m 67.4M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (11m) 19s ago 37m 469M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (21m) 19s ago 38m 529M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (8m) 19s ago 39m 61.4M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (9m) 19s ago 37m 51.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (8m) 19s ago 37m 46.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (11m) 19s ago 33m 16.2M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (11m) 19s ago 33m 16.8M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 19s ago 36m 56.2M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 19s ago 36m 57.6M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 19s ago 36m 49.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (3m) 19s ago 35m 53.5M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a cfe832ae10c3 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (3m) 19s ago 35m 53.1M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a cdf9ea05dcb0 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (2m) 19s ago 34m 55.7M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 408a908ef3cc 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (2m) 19s ago 34m 47.3M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 01f14aea01f3 2024-05-09T04:22:01.669 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (95s) 19s ago 34m 56.7M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a f50fb4f7e63c 2024-05-09T04:22:01.670 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (10m) 19s ago 33m 72.2M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:22:01.670 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (29s) 19s ago 31m 78.9M - 19.0.0-3631-gc66c553a af4a4df59e0a b5f39dcd0e7a 2024-05-09T04:22:01.670 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (23s) 19s ago 31m 78.8M - 19.0.0-3631-gc66c553a af4a4df59e0a 3a8a02e309fe 2024-05-09T04:22:02.315 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-05-09T04:22:02.875 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:02 smithi078 ceph-mon[83176]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-05-09T04:22:02.875 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:02 smithi078 ceph-mon[83176]: from='client.54736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:02.875 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:02 smithi078 ceph-mon[88406]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-05-09T04:22:02.875 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:02 smithi078 ceph-mon[88406]: from='client.54736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:03.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:02 smithi132 ceph-mon[66347]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-05-09T04:22:03.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:02 smithi132 ceph-mon[66347]: from='client.54736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:04.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:03 smithi078 ceph-mon[83176]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:04.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:03 smithi078 ceph-mon[88406]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:04.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:03 smithi132 ceph-mon[66347]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:04.558 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:22:05.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:04 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:22:05.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:04 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2311905635' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:05.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:04 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:22:05.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:04 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/2311905635' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:05.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:04 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:22:05.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:04 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/2311905635' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:05.353 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-05-09T04:22:05.937 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:05 smithi078 ceph-mon[83176]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:05.937 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:05 smithi078 ceph-mon[88406]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:06.154 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:05 smithi132 ceph-mon[66347]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:07.472 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)" 2024-05-09T04:22:08.156 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade status' 2024-05-09T04:22:08.462 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:08 smithi078 ceph-mon[88406]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:08.462 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:08 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/3868314005' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:08.462 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:08 smithi078 ceph-mon[83176]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:08.462 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:08 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/3868314005' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:08.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:08 smithi132 ceph-mon[66347]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:08.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:08 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/3868314005' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:09.501 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:22:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:22:09] "GET /metrics HTTP/1.1" 200 38485 "" "Prometheus/2.43.0" 2024-05-09T04:22:10.280 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:22:10.280 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": null, 2024-05-09T04:22:10.281 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": false, 2024-05-09T04:22:10.281 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "", 2024-05-09T04:22:10.281 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-05-09T04:22:10.281 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": null, 2024-05-09T04:22:10.281 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-05-09T04:22:10.281 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:22:10.281 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:22:10.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:10 smithi132 ceph-mon[66347]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:10.510 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:10 smithi078 ceph-mon[83176]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:10.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:10 smithi078 ceph-mon[88406]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:10.993 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph health detail' 2024-05-09T04:22:11.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:11 smithi078 ceph-mon[83176]: from='client.44811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:11.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:11 smithi078 ceph-mon[88406]: from='client.44811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:11.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:11 smithi132 ceph-mon[66347]: from='client.44811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:12.218 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:12 smithi078 ceph-mon[88406]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:12.219 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:12 smithi078 ceph-mon[83176]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:12.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:12 smithi132 ceph-mon[66347]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:12.994 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-05-09T04:22:13.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:22:13.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:13 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/2919638919' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:22:13.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:22:13.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:13 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/2919638919' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:22:13.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:22:13.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:13 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2919638919' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:22:13.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-09T04:22:14.187 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:14 smithi078 ceph-mon[83176]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:14.188 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:14 smithi078 ceph-mon[88406]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:14.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:14 smithi132 ceph-mon[66347]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:15.454 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:15 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:22:15.454 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:15 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:22:15.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:15 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:22:15.511 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:22:16.266 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-09T04:22:16.276 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi078.front.sepia.ceph.com 2024-05-09T04:22:16.276 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-05-09T04:22:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[88406]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[88406]: from='client.54766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[88406]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:22:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:22:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:22:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:22:16.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:16.762 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[88406]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:22:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[83176]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[83176]: from='client.54766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[83176]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:22:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:22:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:22:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:22:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:16.762 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:16 smithi078 ceph-mon[83176]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:22:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:16 smithi132 ceph-mon[66347]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:16 smithi132 ceph-mon[66347]: from='client.54766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:16 smithi132 ceph-mon[66347]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:22:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:22:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:22:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:22:17.003 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:16 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:17.004 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:16 smithi132 ceph-mon[66347]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c66c553a84eb9faa0029f5f0bb8284e3a826ac85 2024-05-09T04:22:18.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:18 smithi132 ceph-mon[66347]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:18.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:18 smithi078 ceph-mon[83176]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:18.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:18 smithi078 ceph-mon[88406]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:19.234 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:22:19.402 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:22:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:22:19] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-05-09T04:22:19.965 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:22:19.965 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (10m) 37s ago 32m 27.2M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:22:19.965 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (10m) 37s ago 32m 82.0M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:22:19.965 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (12m) 37s ago 32m 67.4M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (11m) 37s ago 37m 469M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (22m) 37s ago 39m 529M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (8m) 37s ago 39m 61.4M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (9m) 37s ago 37m 51.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (8m) 37s ago 37m 46.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (11m) 37s ago 33m 16.2M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (11m) 37s ago 33m 16.8M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 37s ago 37m 56.2M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 37s ago 36m 57.6M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 37s ago 36m 49.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 37s ago 36m 53.5M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a cfe832ae10c3 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (3m) 37s ago 35m 53.1M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a cdf9ea05dcb0 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (3m) 37s ago 35m 55.7M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 408a908ef3cc 2024-05-09T04:22:19.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (2m) 37s ago 34m 47.3M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 01f14aea01f3 2024-05-09T04:22:19.967 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (114s) 37s ago 34m 56.7M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a f50fb4f7e63c 2024-05-09T04:22:19.967 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (11m) 37s ago 33m 72.2M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:22:19.967 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (48s) 37s ago 32m 78.9M - 19.0.0-3631-gc66c553a af4a4df59e0a b5f39dcd0e7a 2024-05-09T04:22:19.967 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (42s) 37s ago 32m 78.8M - 19.0.0-3631-gc66c553a af4a4df59e0a 3a8a02e309fe 2024-05-09T04:22:20.387 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:22:20.387 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:22:20.387 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:22:20.387 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:22:20.387 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:22:20.387 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:22:20.387 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:22:20.387 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:22:20.388 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 8 2024-05-09T04:22:20.388 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:22:20.388 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:22:20.388 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:22:20.388 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:22:20.388 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:22:20.388 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 15 2024-05-09T04:22:20.388 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:22:20.388 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:22:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='client.44829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:22:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:22:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:22:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mgr 2024-05-09T04:22:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.512 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mon 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all crash 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all osd 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all mds 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all rgw 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:22:20.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-09T04:22:20.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:22:20.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='client.54778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:20.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/2128620896' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:20.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='client.44829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:20.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:22:20.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:22:20.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:22:20.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mgr 2024-05-09T04:22:20.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mon 2024-05-09T04:22:20.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all crash 2024-05-09T04:22:20.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all osd 2024-05-09T04:22:20.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all mds 2024-05-09T04:22:20.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all rgw 2024-05-09T04:22:20.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:22:20.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-09T04:22:20.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:22:20.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='client.54778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:20.520 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:20 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/2128620896' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.744 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:22:20.744 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:22:20.744 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:22:20.744 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-09T04:22:20.744 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-05-09T04:22:20.744 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-05-09T04:22:20.744 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-05-09T04:22:20.744 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-05-09T04:22:20.744 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw" 2024-05-09T04:22:20.744 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-05-09T04:22:20.744 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "15/21 daemons upgraded", 2024-05-09T04:22:20.744 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading iscsi daemons", 2024-05-09T04:22:20.745 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-05-09T04:22:20.745 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:22:20.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:20.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='client.44829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:20.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: Upgrade: Target is version 19.0.0-3631-gc66c553a (squid) 2024-05-09T04:22:20.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, digests ['quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082'] 2024-05-09T04:22:20.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mgr 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mon 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all crash 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all osd 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.754 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all mds 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all rgw 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all rbd-mirror 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: Upgrade: Setting container_image for all ceph-exporter 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:20.755 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='client.54778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:20.756 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:20 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2128620896' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:21.161 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-05-09T04:22:21.711 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:21 smithi078 ceph-mon[83176]: from='client.44841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:21.711 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:21 smithi078 ceph-mon[83176]: from='client.44853 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:21.711 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:21 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:21.711 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:21 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:22:21.711 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:21 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:22:21.711 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:21 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/3942665775' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:22:21.711 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:21 smithi078 ceph-mon[88406]: from='client.44841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:21.711 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:21 smithi078 ceph-mon[88406]: from='client.44853 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:21.712 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:21 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:21.712 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:21 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:22:21.712 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:21 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:22:21.712 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:21 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/3942665775' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:22:21.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:21 smithi132 ceph-mon[66347]: from='client.44841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:21.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:21 smithi132 ceph-mon[66347]: from='client.44853 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:21.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:21 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:21.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:21 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi078.vnplzc", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-09T04:22:21.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:21 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:22:21.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:21 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/3942665775' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:22:22.684 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:22 smithi078 ceph-mon[88406]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:22.684 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:22 smithi078 ceph-mon[88406]: Upgrade: Updating iscsi.foo.smithi078.vnplzc 2024-05-09T04:22:22.684 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:22 smithi078 ceph-mon[88406]: Deploying daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:22:22.684 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:22 smithi078 ceph-mon[83176]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:22.684 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:22 smithi078 ceph-mon[83176]: Upgrade: Updating iscsi.foo.smithi078.vnplzc 2024-05-09T04:22:22.684 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:22 smithi078 ceph-mon[83176]: Deploying daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:22:22.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:22 smithi132 ceph-mon[66347]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:22.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:22 smithi132 ceph-mon[66347]: Upgrade: Updating iscsi.foo.smithi078.vnplzc 2024-05-09T04:22:22.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:22 smithi132 ceph-mon[66347]: Deploying daemon iscsi.foo.smithi078.vnplzc on smithi078 2024-05-09T04:22:24.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:24 smithi132 ceph-mon[66347]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:24.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:24 smithi078 ceph-mon[83176]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:24.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:24 smithi078 ceph-mon[88406]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:25.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:25 smithi132 ceph-mon[66347]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:22:25.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:25 smithi078 ceph-mon[83176]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:22:25.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:25 smithi078 ceph-mon[88406]: from='client.15564 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:22:26.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:26 smithi132 ceph-mon[66347]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:26.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:26 smithi078 ceph-mon[83176]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:26.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:26 smithi078 ceph-mon[88406]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:28.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:28 smithi132 ceph-mon[66347]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:28.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:22:28.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:28 smithi078 ceph-mon[83176]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:28.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:22:28.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:28 smithi078 ceph-mon[88406]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:28.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:22:29.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:22:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:22:29] "GET /metrics HTTP/1.1" 200 38482 "" "Prometheus/2.43.0" 2024-05-09T04:22:30.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:30 smithi132 ceph-mon[66347]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:30.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:30 smithi078 ceph-mon[83176]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:30.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:30 smithi078 ceph-mon[88406]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:32.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:32 smithi132 ceph-mon[66347]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:32.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:32 smithi078 ceph-mon[83176]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:32.761 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:32 smithi078 ceph-mon[88406]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:34.740 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:34 smithi078 ceph-mon[83176]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:34.740 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:34 smithi078 ceph-mon[88406]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:34.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:34 smithi132 ceph-mon[66347]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:22:35.011 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:22:34 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:22:34.800+0000 7f80b8e96700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi078.vnplzc on host smithi078 failed. 2024-05-09T04:22:35.897 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[88406]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi078.vnplzc on host smithi078 failed. 2024-05-09T04:22:35.897 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:35.897 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:22:35.897 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[88406]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:35.897 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:22:35.897 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:22:35.897 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:35.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[83176]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi078.vnplzc on host smithi078 failed. 2024-05-09T04:22:35.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:35.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:22:35.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[83176]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:35.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:22:35.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:22:35.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:36.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:35 smithi132 ceph-mon[66347]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi078.vnplzc on host smithi078 failed. 2024-05-09T04:22:36.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:36.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:22:36.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:35 smithi132 ceph-mon[66347]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:36.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:22:36.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:22:36.254 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:37.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:36 smithi132 ceph-mon[66347]: Health check failed: Upgrading daemon iscsi.foo.smithi078.vnplzc on host smithi078 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-05-09T04:22:37.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:36 smithi078 ceph-mon[83176]: Health check failed: Upgrading daemon iscsi.foo.smithi078.vnplzc on host smithi078 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-05-09T04:22:37.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:36 smithi078 ceph-mon[88406]: Health check failed: Upgrading daemon iscsi.foo.smithi078.vnplzc on host smithi078 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-05-09T04:22:38.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:37 smithi132 ceph-mon[66347]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 783 B/s rd, 0 op/s 2024-05-09T04:22:38.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:37 smithi078 ceph-mon[83176]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 783 B/s rd, 0 op/s 2024-05-09T04:22:38.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:37 smithi078 ceph-mon[88406]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 783 B/s rd, 0 op/s 2024-05-09T04:22:39.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:22:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:22:39] "GET /metrics HTTP/1.1" 200 38482 "" "Prometheus/2.43.0" 2024-05-09T04:22:40.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:39 smithi132 ceph-mon[66347]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 783 B/s rd, 0 op/s 2024-05-09T04:22:40.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:39 smithi078 ceph-mon[88406]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 783 B/s rd, 0 op/s 2024-05-09T04:22:40.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:39 smithi078 ceph-mon[83176]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 783 B/s rd, 0 op/s 2024-05-09T04:22:42.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:41 smithi132 ceph-mon[66347]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 348 B/s rd, 0 op/s 2024-05-09T04:22:42.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:41 smithi078 ceph-mon[88406]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 348 B/s rd, 0 op/s 2024-05-09T04:22:42.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:41 smithi078 ceph-mon[83176]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 348 B/s rd, 0 op/s 2024-05-09T04:22:44.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:43 smithi078 ceph-mon[83176]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 348 B/s rd, 0 op/s 2024-05-09T04:22:44.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:44.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:22:44.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:43 smithi078 ceph-mon[88406]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 348 B/s rd, 0 op/s 2024-05-09T04:22:44.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:44.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:22:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:43 smithi132 ceph-mon[66347]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 348 B/s rd, 0 op/s 2024-05-09T04:22:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:22:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:22:46.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:45 smithi132 ceph-mon[66347]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 348 B/s rd, 0 op/s 2024-05-09T04:22:46.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:45 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2538635165' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:22:46.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:45 smithi078 ceph-mon[83176]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 348 B/s rd, 0 op/s 2024-05-09T04:22:46.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:45 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/2538635165' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:22:46.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:45 smithi078 ceph-mon[88406]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 348 B/s rd, 0 op/s 2024-05-09T04:22:46.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:45 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/2538635165' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-09T04:22:47.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:46 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/1853245345' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1828432941"}]: dispatch 2024-05-09T04:22:47.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:46 smithi132 ceph-mon[66347]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1828432941"}]: dispatch 2024-05-09T04:22:47.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:46 smithi132 ceph-mon[66347]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:22:47.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:46 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/1853245345' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1828432941"}]: dispatch 2024-05-09T04:22:47.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:46 smithi078 ceph-mon[83176]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1828432941"}]: dispatch 2024-05-09T04:22:47.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:46 smithi078 ceph-mon[83176]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:22:47.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:46 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/1853245345' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1828432941"}]: dispatch 2024-05-09T04:22:47.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:46 smithi078 ceph-mon[88406]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1828432941"}]: dispatch 2024-05-09T04:22:47.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:46 smithi078 ceph-mon[88406]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:22:48.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:47 smithi132 ceph-mon[66347]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1828432941"}]': finished 2024-05-09T04:22:48.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:47 smithi132 ceph-mon[66347]: osdmap e155: 8 total, 8 up, 8 in 2024-05-09T04:22:48.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:47 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2437420869' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2043811703"}]: dispatch 2024-05-09T04:22:48.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:47 smithi132 ceph-mon[66347]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2043811703"}]: dispatch 2024-05-09T04:22:48.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:47 smithi078 ceph-mon[88406]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1828432941"}]': finished 2024-05-09T04:22:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:47 smithi078 ceph-mon[88406]: osdmap e155: 8 total, 8 up, 8 in 2024-05-09T04:22:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:47 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/2437420869' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2043811703"}]: dispatch 2024-05-09T04:22:48.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:47 smithi078 ceph-mon[88406]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2043811703"}]: dispatch 2024-05-09T04:22:48.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:47 smithi078 ceph-mon[83176]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/1828432941"}]': finished 2024-05-09T04:22:48.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:47 smithi078 ceph-mon[83176]: osdmap e155: 8 total, 8 up, 8 in 2024-05-09T04:22:48.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:47 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/2437420869' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2043811703"}]: dispatch 2024-05-09T04:22:48.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:47 smithi078 ceph-mon[83176]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2043811703"}]: dispatch 2024-05-09T04:22:49.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:48 smithi132 ceph-mon[66347]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2043811703"}]': finished 2024-05-09T04:22:49.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:48 smithi132 ceph-mon[66347]: osdmap e156: 8 total, 8 up, 8 in 2024-05-09T04:22:49.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:48 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/581952473' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2070162107"}]: dispatch 2024-05-09T04:22:49.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:48 smithi132 ceph-mon[66347]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2070162107"}]: dispatch 2024-05-09T04:22:49.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:48 smithi132 ceph-mon[66347]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:22:49.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:22:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:22:49] "GET /metrics HTTP/1.1" 200 38563 "" "Prometheus/2.43.0" 2024-05-09T04:22:49.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:48 smithi078 ceph-mon[88406]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2043811703"}]': finished 2024-05-09T04:22:49.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:48 smithi078 ceph-mon[88406]: osdmap e156: 8 total, 8 up, 8 in 2024-05-09T04:22:49.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:48 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/581952473' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2070162107"}]: dispatch 2024-05-09T04:22:49.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:48 smithi078 ceph-mon[88406]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2070162107"}]: dispatch 2024-05-09T04:22:49.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:48 smithi078 ceph-mon[88406]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:22:49.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:48 smithi078 ceph-mon[83176]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6800/2043811703"}]': finished 2024-05-09T04:22:49.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:48 smithi078 ceph-mon[83176]: osdmap e156: 8 total, 8 up, 8 in 2024-05-09T04:22:49.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:48 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/581952473' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2070162107"}]: dispatch 2024-05-09T04:22:49.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:48 smithi078 ceph-mon[83176]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2070162107"}]: dispatch 2024-05-09T04:22:49.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:48 smithi078 ceph-mon[83176]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-09T04:22:50.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:49 smithi132 ceph-mon[66347]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2070162107"}]': finished 2024-05-09T04:22:50.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:49 smithi132 ceph-mon[66347]: osdmap e157: 8 total, 8 up, 8 in 2024-05-09T04:22:50.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:49 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2159323811' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4266719719"}]: dispatch 2024-05-09T04:22:50.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:49 smithi132 ceph-mon[66347]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4266719719"}]: dispatch 2024-05-09T04:22:50.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:49 smithi078 ceph-mon[83176]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2070162107"}]': finished 2024-05-09T04:22:50.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:49 smithi078 ceph-mon[83176]: osdmap e157: 8 total, 8 up, 8 in 2024-05-09T04:22:50.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:49 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/2159323811' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4266719719"}]: dispatch 2024-05-09T04:22:50.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:49 smithi078 ceph-mon[83176]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4266719719"}]: dispatch 2024-05-09T04:22:50.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:49 smithi078 ceph-mon[88406]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2070162107"}]': finished 2024-05-09T04:22:50.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:49 smithi078 ceph-mon[88406]: osdmap e157: 8 total, 8 up, 8 in 2024-05-09T04:22:50.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:49 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/2159323811' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4266719719"}]: dispatch 2024-05-09T04:22:50.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:49 smithi078 ceph-mon[88406]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4266719719"}]: dispatch 2024-05-09T04:22:51.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:50 smithi132 ceph-mon[66347]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4266719719"}]': finished 2024-05-09T04:22:51.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:50 smithi132 ceph-mon[66347]: osdmap e158: 8 total, 8 up, 8 in 2024-05-09T04:22:51.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:50 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2384559994' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2043811703"}]: dispatch 2024-05-09T04:22:51.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:50 smithi132 ceph-mon[66347]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2043811703"}]: dispatch 2024-05-09T04:22:51.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:50 smithi132 ceph-mon[66347]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-09T04:22:51.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:50 smithi078 ceph-mon[83176]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4266719719"}]': finished 2024-05-09T04:22:51.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:50 smithi078 ceph-mon[83176]: osdmap e158: 8 total, 8 up, 8 in 2024-05-09T04:22:51.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:50 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/2384559994' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2043811703"}]: dispatch 2024-05-09T04:22:51.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:50 smithi078 ceph-mon[83176]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2043811703"}]: dispatch 2024-05-09T04:22:51.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:50 smithi078 ceph-mon[83176]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-09T04:22:51.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:50 smithi078 ceph-mon[88406]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/4266719719"}]': finished 2024-05-09T04:22:51.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:50 smithi078 ceph-mon[88406]: osdmap e158: 8 total, 8 up, 8 in 2024-05-09T04:22:51.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:50 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/2384559994' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2043811703"}]: dispatch 2024-05-09T04:22:51.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:50 smithi078 ceph-mon[88406]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2043811703"}]: dispatch 2024-05-09T04:22:51.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:50 smithi078 ceph-mon[88406]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-09T04:22:51.523 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:22:51.861 INFO:teuthology.orchestra.run.smithi078.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi078.vnplzc on host smithi078 failed." 2024-05-09T04:22:52.163 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:51 smithi078 ceph-mon[88406]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2043811703"}]': finished 2024-05-09T04:22:52.163 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:51 smithi078 ceph-mon[88406]: osdmap e159: 8 total, 8 up, 8 in 2024-05-09T04:22:52.163 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:51 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/4274619217' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2181898287"}]: dispatch 2024-05-09T04:22:52.163 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:51 smithi078 ceph-mon[88406]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2181898287"}]: dispatch 2024-05-09T04:22:52.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:51 smithi078 ceph-mon[88406]: from='client.54868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:52.164 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:51 smithi078 ceph-mon[83176]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2043811703"}]': finished 2024-05-09T04:22:52.164 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:51 smithi078 ceph-mon[83176]: osdmap e159: 8 total, 8 up, 8 in 2024-05-09T04:22:52.164 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:51 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/4274619217' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2181898287"}]: dispatch 2024-05-09T04:22:52.164 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:51 smithi078 ceph-mon[83176]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2181898287"}]: dispatch 2024-05-09T04:22:52.164 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:51 smithi078 ceph-mon[83176]: from='client.54868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:52.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:51 smithi132 ceph-mon[66347]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:6801/2043811703"}]': finished 2024-05-09T04:22:52.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:51 smithi132 ceph-mon[66347]: osdmap e159: 8 total, 8 up, 8 in 2024-05-09T04:22:52.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:51 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/4274619217' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2181898287"}]: dispatch 2024-05-09T04:22:52.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:51 smithi132 ceph-mon[66347]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2181898287"}]: dispatch 2024-05-09T04:22:52.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:51 smithi132 ceph-mon[66347]: from='client.54868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:52.574 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T04:22:53.139 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:52 smithi078 ceph-mon[88406]: from='client.54874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:53.139 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:52 smithi078 ceph-mon[88406]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2181898287"}]': finished 2024-05-09T04:22:53.139 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:52 smithi078 ceph-mon[88406]: osdmap e160: 8 total, 8 up, 8 in 2024-05-09T04:22:53.139 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:52 smithi078 ceph-mon[88406]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:22:53.139 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:52 smithi078 ceph-mon[83176]: from='client.54874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:53.139 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:52 smithi078 ceph-mon[83176]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2181898287"}]': finished 2024-05-09T04:22:53.139 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:52 smithi078 ceph-mon[83176]: osdmap e160: 8 total, 8 up, 8 in 2024-05-09T04:22:53.140 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:52 smithi078 ceph-mon[83176]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:22:53.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:52 smithi132 ceph-mon[66347]: from='client.54874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:53.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:52 smithi132 ceph-mon[66347]: from='client.? ' entity='client.iscsi.foo.smithi078.vnplzc' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.78:0/2181898287"}]': finished 2024-05-09T04:22:53.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:52 smithi132 ceph-mon[66347]: osdmap e160: 8 total, 8 up, 8 in 2024-05-09T04:22:53.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:52 smithi132 ceph-mon[66347]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail 2024-05-09T04:22:54.608 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:22:54.608 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (11m) 72s ago 33m 27.2M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:22:54.608 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (11m) 72s ago 33m 82.0M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (12m) 72s ago 32m 67.4M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (12m) 72s ago 38m 469M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (22m) 72s ago 39m 529M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (9m) 72s ago 39m 61.4M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (10m) 72s ago 38m 51.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (8m) 72s ago 38m 46.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (12m) 72s ago 34m 16.2M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (12m) 72s ago 34m 16.8M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (7m) 72s ago 37m 56.2M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 72s ago 37m 57.6M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 72s ago 37m 49.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 72s ago 36m 53.5M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a cfe832ae10c3 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (4m) 72s ago 36m 53.1M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a cdf9ea05dcb0 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (3m) 72s ago 35m 55.7M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 408a908ef3cc 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (3m) 72s ago 35m 47.3M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 01f14aea01f3 2024-05-09T04:22:54.609 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (2m) 72s ago 35m 56.7M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a f50fb4f7e63c 2024-05-09T04:22:54.610 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (11m) 72s ago 34m 72.2M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:22:54.610 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (82s) 72s ago 32m 78.9M - 19.0.0-3631-gc66c553a af4a4df59e0a b5f39dcd0e7a 2024-05-09T04:22:54.610 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (76s) 72s ago 32m 78.8M - 19.0.0-3631-gc66c553a af4a4df59e0a 3a8a02e309fe 2024-05-09T04:22:55.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:54 smithi132 ceph-mon[66347]: from='client.44937 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:55.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:54 smithi132 ceph-mon[66347]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 859 B/s rd, 0 op/s 2024-05-09T04:22:55.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:54 smithi078 ceph-mon[83176]: from='client.44937 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:55.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:54 smithi078 ceph-mon[83176]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 859 B/s rd, 0 op/s 2024-05-09T04:22:55.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:54 smithi078 ceph-mon[88406]: from='client.44937 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:22:55.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:54 smithi078 ceph-mon[88406]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 859 B/s rd, 0 op/s 2024-05-09T04:22:55.351 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions' 2024-05-09T04:22:55.972 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:55 smithi078 ceph-mon[83176]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:22:55.973 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:55 smithi078 ceph-mon[88406]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:22:56.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:55 smithi132 ceph-mon[66347]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:22:57.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:56 smithi078 ceph-mon[83176]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 736 B/s rd, 0 op/s 2024-05-09T04:22:57.127 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:56 smithi078 ceph-mon[88406]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 736 B/s rd, 0 op/s 2024-05-09T04:22:57.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:56 smithi132 ceph-mon[66347]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 736 B/s rd, 0 op/s 2024-05-09T04:22:57.666 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:22:57.666 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:22:57.666 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:22:57.666 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:22:57.666 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:22:57.666 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:22:57.666 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:22:57.666 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:22:57.666 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 8 2024-05-09T04:22:57.666 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:22:57.666 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:22:57.666 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:22:57.667 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:22:57.667 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:22:57.667 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 15 2024-05-09T04:22:57.667 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:22:57.667 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:22:57.998 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:57 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/2492169246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:57.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:57 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/2492169246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:58.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:57 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2492169246' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:22:58.417 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-05-09T04:22:58.996 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:22:58.996 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:22:58 smithi078 ceph-mon[83176]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:58.997 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:22:58.997 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:22:58 smithi078 ceph-mon[88406]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:59.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:22:59.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:22:58 smithi132 ceph-mon[66347]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:22:59.254 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:22:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:22:59] "GET /metrics HTTP/1.1" 200 38563 "" "Prometheus/2.43.0" 2024-05-09T04:23:00.244 INFO:teuthology.orchestra.run.smithi078.stdout:wait for servicemap items w/ changing names to refresh 2024-05-09T04:23:01.052 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'sleep 60' 2024-05-09T04:23:01.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:00 smithi132 ceph-mon[66347]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-09T04:23:01.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:00 smithi078 ceph-mon[83176]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-09T04:23:01.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:00 smithi078 ceph-mon[88406]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-09T04:23:03.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:02 smithi078 ceph-mon[83176]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 936 B/s rd, 0 op/s 2024-05-09T04:23:03.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:02 smithi078 ceph-mon[88406]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 936 B/s rd, 0 op/s 2024-05-09T04:23:03.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:02 smithi132 ceph-mon[66347]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 936 B/s rd, 0 op/s 2024-05-09T04:23:05.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:04 smithi132 ceph-mon[66347]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:05.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:04 smithi078 ceph-mon[83176]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:05.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:04 smithi078 ceph-mon[88406]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:06.154 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:05 smithi132 ceph-mon[66347]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:06.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:05 smithi078 ceph-mon[83176]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:06.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:05 smithi078 ceph-mon[88406]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:07.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:06 smithi132 ceph-mon[66347]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:07.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:06 smithi078 ceph-mon[83176]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:07.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:06 smithi078 ceph-mon[88406]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:09.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:08 smithi132 ceph-mon[66347]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:09.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:08 smithi078 ceph-mon[83176]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:09.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:08 smithi078 ceph-mon[88406]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:09.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:23:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:23:09] "GET /metrics HTTP/1.1" 200 38563 "" "Prometheus/2.43.0" 2024-05-09T04:23:11.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:10 smithi132 ceph-mon[66347]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:11.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:10 smithi078 ceph-mon[83176]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:11.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:10 smithi078 ceph-mon[88406]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:13.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:12 smithi132 ceph-mon[66347]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:13.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:12 smithi078 ceph-mon[83176]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:13.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:12 smithi078 ceph-mon[88406]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:14.162 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:23:14.162 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:23:14.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:23:15.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:14 smithi132 ceph-mon[66347]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:15.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:14 smithi078 ceph-mon[88406]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:15.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:14 smithi078 ceph-mon[83176]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:16.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:15 smithi132 ceph-mon[66347]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:16.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:15 smithi078 ceph-mon[83176]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:16.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:15 smithi078 ceph-mon[88406]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:17.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:16 smithi132 ceph-mon[66347]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:17.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:16 smithi078 ceph-mon[83176]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:17.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:16 smithi078 ceph-mon[88406]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:19.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:18 smithi132 ceph-mon[66347]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:19.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:18 smithi078 ceph-mon[88406]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:19.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:18 smithi078 ceph-mon[83176]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:19.262 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:23:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:23:19] "GET /metrics HTTP/1.1" 200 38565 "" "Prometheus/2.43.0" 2024-05-09T04:23:21.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:20 smithi132 ceph-mon[66347]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:21.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:20 smithi078 ceph-mon[83176]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:21.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:20 smithi078 ceph-mon[88406]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:23.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:22 smithi132 ceph-mon[66347]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:23.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:22 smithi078 ceph-mon[83176]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:23.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:22 smithi078 ceph-mon[88406]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:25.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:24 smithi132 ceph-mon[66347]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:25.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:24 smithi078 ceph-mon[83176]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:25.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:24 smithi078 ceph-mon[88406]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:26.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:25 smithi132 ceph-mon[66347]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:26.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:25 smithi078 ceph-mon[83176]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:26.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:25 smithi078 ceph-mon[88406]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:27.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:26 smithi132 ceph-mon[66347]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:27.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:26 smithi078 ceph-mon[83176]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:27.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:26 smithi078 ceph-mon[88406]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:28.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:23:28.510 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:23:28.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:23:29.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:29 smithi132 ceph-mon[66347]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:29.510 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:29 smithi078 ceph-mon[88406]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:29.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:29 smithi078 ceph-mon[83176]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:29.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:23:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:23:29] "GET /metrics HTTP/1.1" 200 38566 "" "Prometheus/2.43.0" 2024-05-09T04:23:31.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:30 smithi132 ceph-mon[66347]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:31.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:30 smithi078 ceph-mon[83176]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:31.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:30 smithi078 ceph-mon[88406]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:33.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:32 smithi132 ceph-mon[66347]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:33.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:32 smithi078 ceph-mon[83176]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:33.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:32 smithi078 ceph-mon[88406]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:35.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:34 smithi132 ceph-mon[66347]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:35.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:34 smithi078 ceph-mon[83176]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:35.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:34 smithi078 ceph-mon[88406]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:36.154 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:23:36.154 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:35 smithi132 ceph-mon[66347]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:36.154 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:23:36.154 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:23:36.155 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:35 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:23:36.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:23:36.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:35 smithi078 ceph-mon[83176]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:36.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:23:36.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:23:36.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:35 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:23:36.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-09T04:23:36.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:35 smithi078 ceph-mon[88406]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:36.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-09T04:23:36.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-09T04:23:36.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:35 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' 2024-05-09T04:23:37.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:36 smithi132 ceph-mon[66347]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:37.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:36 smithi078 ceph-mon[83176]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:37.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:36 smithi078 ceph-mon[88406]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:39.252 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:38 smithi132 ceph-mon[66347]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:39.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:38 smithi078 ceph-mon[83176]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:39.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:38 smithi078 ceph-mon[88406]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:39.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:23:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:23:39] "GET /metrics HTTP/1.1" 200 38566 "" "Prometheus/2.43.0" 2024-05-09T04:23:41.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:40 smithi132 ceph-mon[66347]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:41.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:40 smithi078 ceph-mon[83176]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:41.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:40 smithi078 ceph-mon[88406]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:43.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:42 smithi132 ceph-mon[66347]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:43.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:42 smithi078 ceph-mon[83176]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:43.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:42 smithi078 ceph-mon[88406]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:44.162 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:43 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:23:44.163 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:43 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:23:44.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:43 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:23:45.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:44 smithi132 ceph-mon[66347]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:45.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:44 smithi078 ceph-mon[83176]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:45.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:44 smithi078 ceph-mon[88406]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:46.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:45 smithi132 ceph-mon[66347]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:46.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:45 smithi078 ceph-mon[83176]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:46.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:45 smithi078 ceph-mon[88406]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:47.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:46 smithi132 ceph-mon[66347]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:47.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:46 smithi078 ceph-mon[83176]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:47.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:46 smithi078 ceph-mon[88406]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:49.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:48 smithi132 ceph-mon[66347]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:49.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:48 smithi078 ceph-mon[83176]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:49.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:48 smithi078 ceph-mon[88406]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:49.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:23:49 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:23:49] "GET /metrics HTTP/1.1" 200 38568 "" "Prometheus/2.43.0" 2024-05-09T04:23:51.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:50 smithi132 ceph-mon[66347]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:51.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:50 smithi078 ceph-mon[83176]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:51.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:50 smithi078 ceph-mon[88406]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:53.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:52 smithi132 ceph-mon[66347]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:53.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:52 smithi078 ceph-mon[83176]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:53.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:52 smithi078 ceph-mon[88406]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:55.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:54 smithi132 ceph-mon[66347]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:55.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:54 smithi078 ceph-mon[88406]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:55.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:54 smithi078 ceph-mon[83176]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:56.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:55 smithi132 ceph-mon[66347]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:56.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:55 smithi078 ceph-mon[88406]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:56.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:55 smithi078 ceph-mon[83176]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:23:57.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:56 smithi132 ceph-mon[66347]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:57.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:56 smithi078 ceph-mon[88406]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:57.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:56 smithi078 ceph-mon[83176]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:23:58.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:58 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:23:58.510 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:58 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:23:58.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:58 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:23:59.503 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:23:59 smithi132 ceph-mon[66347]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:59.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:23:59 smithi078 ceph-mon[83176]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:59.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:23:59 smithi078 ceph-mon[88406]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:23:59.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:23:59 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:23:59] "GET /metrics HTTP/1.1" 200 38565 "" "Prometheus/2.43.0" 2024-05-09T04:24:01.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:00 smithi132 ceph-mon[66347]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:01.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:00 smithi078 ceph-mon[83176]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:01.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:00 smithi078 ceph-mon[88406]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:03.159 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:02 smithi078 ceph-mon[88406]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:03.159 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:02 smithi078 ceph-mon[83176]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:03.252 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:02 smithi132 ceph-mon[66347]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:03.538 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ps' 2024-05-09T04:24:05.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:04 smithi078 ceph-mon[83176]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:05.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:04 smithi078 ceph-mon[88406]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:05.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:04 smithi132 ceph-mon[66347]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:05.464 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-09T04:24:05.464 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (12m) 2m ago 34m 27.2M - 0.25.0 c8568f914cd2 3cd8b265bc1d 2024-05-09T04:24:05.464 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi132 *:3000 running (12m) 2m ago 34m 82.0M - 9.4.12 2bacad6d85d8 663d68ed82f1 2024-05-09T04:24:05.464 INFO:teuthology.orchestra.run.smithi078.stdout:iscsi.foo.smithi078.vnplzc smithi078 running (13m) 2m ago 33m 67.4M - 3.5 e1d6a67b021e f4e6beae9da1 2024-05-09T04:24:05.464 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi132 *:8443,9283,8765 running (13m) 2m ago 39m 469M - 19.0.0-3631-gc66c553a af4a4df59e0a 0569600c45d0 2024-05-09T04:24:05.465 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (23m) 2m ago 41m 529M - 19.0.0-3631-gc66c553a af4a4df59e0a cc8cc8e980a8 2024-05-09T04:24:05.465 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (10m) 2m ago 41m 61.4M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a ffed80c876d1 2024-05-09T04:24:05.465 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi132 running (11m) 2m ago 39m 51.7M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a d9d8fb1d89f0 2024-05-09T04:24:05.465 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (10m) 2m ago 39m 46.9M 2048M 19.0.0-3631-gc66c553a af4a4df59e0a f45be4f14efd 2024-05-09T04:24:05.465 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (13m) 2m ago 35m 16.2M - 1.5.0 0da6a335fe13 c0004282e7ce 2024-05-09T04:24:05.465 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi132 *:9100 running (13m) 2m ago 35m 16.8M - 1.5.0 0da6a335fe13 71aa9559f36f 2024-05-09T04:24:05.465 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 2m ago 38m 56.2M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a ba788dd1e3fe 2024-05-09T04:24:05.465 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 2m ago 38m 57.6M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a c1e61031cb25 2024-05-09T04:24:05.465 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 2m ago 38m 49.8M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a 65617f5ef4ae 2024-05-09T04:24:05.465 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (6m) 2m ago 37m 53.5M 2992M 19.0.0-3631-gc66c553a af4a4df59e0a cfe832ae10c3 2024-05-09T04:24:05.465 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi132 running (5m) 2m ago 37m 53.1M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a cdf9ea05dcb0 2024-05-09T04:24:05.465 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi132 running (4m) 2m ago 37m 55.7M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 408a908ef3cc 2024-05-09T04:24:05.466 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi132 running (4m) 2m ago 36m 47.3M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a 01f14aea01f3 2024-05-09T04:24:05.466 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi132 running (3m) 2m ago 36m 56.7M 3248M 19.0.0-3631-gc66c553a af4a4df59e0a f50fb4f7e63c 2024-05-09T04:24:05.466 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi132 *:9095 running (12m) 2m ago 35m 72.2M - 2.43.0 a07b618ecd1d c02ff059e704 2024-05-09T04:24:05.466 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi078.bfidtv smithi078 *:8000 running (2m) 2m ago 33m 78.9M - 19.0.0-3631-gc66c553a af4a4df59e0a b5f39dcd0e7a 2024-05-09T04:24:05.466 INFO:teuthology.orchestra.run.smithi078.stdout:rgw.foo.smithi132.pcutwm smithi132 *:8000 running (2m) 2m ago 33m 78.8M - 19.0.0-3631-gc66c553a af4a4df59e0a 3a8a02e309fe 2024-05-09T04:24:06.140 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:05 smithi078 ceph-mon[88406]: from='client.54892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:06.140 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:05 smithi078 ceph-mon[88406]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:24:06.140 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:05 smithi078 ceph-mon[83176]: from='client.54892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:06.140 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:05 smithi078 ceph-mon[83176]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:24:06.153 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:05 smithi132 ceph-mon[66347]: from='client.54892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:06.153 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:05 smithi132 ceph-mon[66347]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:24:06.165 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions' 2024-05-09T04:24:07.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:06 smithi132 ceph-mon[66347]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:07.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:06 smithi078 ceph-mon[88406]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:07.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:06 smithi078 ceph-mon[83176]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:08.268 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:24:08.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-05-09T04:24:08.268 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 3 2024-05-09T04:24:08.268 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:24:08.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-05-09T04:24:08.268 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:24:08.268 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:24:08.268 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-05-09T04:24:08.269 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 8 2024-05-09T04:24:08.269 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:24:08.269 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw": { 2024-05-09T04:24:08.269 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 2 2024-05-09T04:24:08.269 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-05-09T04:24:08.269 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-05-09T04:24:08.269 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)": 15 2024-05-09T04:24:08.269 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-05-09T04:24:08.269 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:24:08.620 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:08 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/2753314831' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:24:08.620 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:08 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/2753314831' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:24:08.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:08 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/2753314831' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:24:08.962 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch upgrade status' 2024-05-09T04:24:09.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:09 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:24:09] "GET /metrics HTTP/1.1" 200 38565 "" "Prometheus/2.43.0" 2024-05-09T04:24:09.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:09 smithi078 ceph-mon[88406]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:09.599 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:09 smithi078 ceph-mon[83176]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:09.753 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:09 smithi132 ceph-mon[66347]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:10.971 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:24:10.972 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082", 2024-05-09T04:24:10.972 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-05-09T04:24:10.972 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-09T04:24:10.972 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-05-09T04:24:10.972 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-05-09T04:24:10.972 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-05-09T04:24:10.972 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-05-09T04:24:10.972 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw" 2024-05-09T04:24:10.972 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-05-09T04:24:10.972 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "15/21 daemons upgraded", 2024-05-09T04:24:10.972 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi078.vnplzc on host smithi078 failed.", 2024-05-09T04:24:10.972 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": true 2024-05-09T04:24:10.973 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:24:11.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:10 smithi132 ceph-mon[66347]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:11.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:10 smithi078 ceph-mon[83176]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:11.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:10 smithi078 ceph-mon[88406]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:11.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph health detail' 2024-05-09T04:24:12.212 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:11 smithi078 ceph-mon[83176]: from='client.44961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:12.213 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:11 smithi078 ceph-mon[88406]: from='client.44961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:12.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:11 smithi132 ceph-mon[66347]: from='client.44961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:13.161 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:12 smithi078 ceph-mon[83176]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:13.161 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:12 smithi078 ceph-mon[88406]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:13.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:12 smithi132 ceph-mon[66347]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:13.571 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN Upgrading daemon iscsi.foo.smithi078.vnplzc on host smithi078 failed. 2024-05-09T04:24:13.571 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi078.vnplzc on host smithi078 failed. 2024-05-09T04:24:13.571 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrade daemon: iscsi.foo.smithi078.vnplzc: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi078.vnplzc ... 2024-05-09T04:24:13.571 INFO:teuthology.orchestra.run.smithi078.stdout:Creating ceph-iscsi config... 2024-05-09T04:24:13.571 INFO:teuthology.orchestra.run.smithi078.stdout:Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/iscsi-gateway.cfg 2024-05-09T04:24:13.571 INFO:teuthology.orchestra.run.smithi078.stdout:Write file: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/iscsi.foo.smithi078.vnplzc/tcmu-runner-entrypoint.sh 2024-05-09T04:24:13.571 INFO:teuthology.orchestra.run.smithi078.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2d2777c79e\x2d0db6\x2d11ef\x2dbc97\x2dc7b262605968.slice/ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service 2024-05-09T04:24:13.571 INFO:teuthology.orchestra.run.smithi078.stdout:Non-zero exit code 1 from systemctl start ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout:systemctl: stderr Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout:systemctl: stderr See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout:Traceback (most recent call last): 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout: return _run_code(code, main_globals, None, 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout: exec(code, run_globals) 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5647, in 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 5635, in main 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3117, in command_deploy_from 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3152, in _common_deploy 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 3172, in _deploy_daemon_container 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1087, in deploy_daemon 2024-05-09T04:24:13.572 INFO:teuthology.orchestra.run.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/__main__.py", line 1223, in deploy_daemon_units 2024-05-09T04:24:13.573 INFO:teuthology.orchestra.run.smithi078.stdout: File "/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/cephadm.fd3782cbb5f0efd7733925e91af2ba100c48e49eb2cca6c4922dddd661e8f568/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-09T04:24:13.573 INFO:teuthology.orchestra.run.smithi078.stdout:RuntimeError: Failed command: systemctl start ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc: Job for ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service failed because the control process exited with error code. 2024-05-09T04:24:13.573 INFO:teuthology.orchestra.run.smithi078.stdout:See "systemctl status ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" and "journalctl -xeu ceph-2777c79e-0db6-11ef-bc97-c7b262605968@iscsi.foo.smithi078.vnplzc.service" for details. 2024-05-09T04:24:14.173 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:13 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:24:14.173 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:13 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/807252225' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:24:14.173 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:13 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:24:14.173 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:13 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/807252225' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:24:14.197 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-05-09T04:24:14.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:13 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:24:14.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:13 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/807252225' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-09T04:24:15.199 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:14 smithi078 ceph-mon[88406]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:15.199 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:14 smithi078 ceph-mon[83176]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:15.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:14 smithi132 ceph-mon[66347]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:15.990 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:15 smithi078 ceph-mon[88406]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:24:15.990 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:15 smithi078 ceph-mon[83176]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:24:16.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:15 smithi132 ceph-mon[66347]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:24:16.322 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-05-09T04:24:16.973 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-05-09T04:24:17.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:16 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/4138413552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:24:17.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:16 smithi132 ceph-mon[66347]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:17.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:16 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/4138413552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:24:17.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:16 smithi078 ceph-mon[88406]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:17.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:16 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/4138413552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:24:17.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:16 smithi078 ceph-mon[83176]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:19.068 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3631-gc66c553a (c66c553a84eb9faa0029f5f0bb8284e3a826ac85) squid (dev)" 2024-05-09T04:24:19.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:18 smithi132 ceph-mon[66347]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:19.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:18 smithi078 ceph-mon[83176]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:19.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:18 smithi078 ceph-mon[88406]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:19.261 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:19 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:24:19] "GET /metrics HTTP/1.1" 200 38564 "" "Prometheus/2.43.0" 2024-05-09T04:24:19.646 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -e sha1=c66c553a84eb9faa0029f5f0bb8284e3a826ac85 -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-05-09T04:24:20.208 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:19 smithi078 ceph-mon[83176]: from='client.? 172.21.15.78:0/4206229088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:24:20.208 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:19 smithi078 ceph-mon[88406]: from='client.? 172.21.15.78:0/4206229088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:24:20.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:19 smithi132 ceph-mon[66347]: from='client.? 172.21.15.78:0/4206229088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-09T04:24:21.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:20 smithi078 ceph-mon[83176]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:21.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:20 smithi078 ceph-mon[88406]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:21.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:20 smithi132 ceph-mon[66347]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:21.770 INFO:teuthology.orchestra.run.smithi078.stdout:osd 8 2m ago - 2024-05-09T04:24:22.094 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:21 smithi078 ceph-mon[83176]: from='client.54928 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:22.095 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:21 smithi078 ceph-mon[88406]: from='client.54928 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:22.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:21 smithi132 ceph-mon[66347]: from='client.54928 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:22.387 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-09T04:24:22.397 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi078.front.sepia.ceph.com 2024-05-09T04:24:22.398 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-05-09T04:24:23.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:22 smithi078 ceph-mon[83176]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:23.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:22 smithi078 ceph-mon[88406]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:23.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:22 smithi132 ceph-mon[66347]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:25.077 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-09T04:24:25.078 INFO:teuthology.orchestra.run.smithi078.stdout: "image": "quay.io/ceph/ceph", 2024-05-09T04:24:25.078 INFO:teuthology.orchestra.run.smithi078.stdout: "registry": "quay.io", 2024-05-09T04:24:25.078 INFO:teuthology.orchestra.run.smithi078.stdout: "bare_image": "ceph/ceph", 2024-05-09T04:24:25.078 INFO:teuthology.orchestra.run.smithi078.stdout: "versions": [] 2024-05-09T04:24:25.078 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-09T04:24:25.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:24 smithi132 ceph-mon[66347]: from='client.44991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:25.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:24 smithi132 ceph-mon[66347]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:25.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:24 smithi078 ceph-mon[83176]: from='client.44991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:25.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:24 smithi078 ceph-mon[83176]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:25.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:24 smithi078 ceph-mon[88406]: from='client.44991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:25.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:24 smithi078 ceph-mon[88406]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:25.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-05-09T04:24:26.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:25 smithi078 ceph-mon[88406]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:24:26.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:25 smithi078 ceph-mon[83176]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:24:26.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:25 smithi132 ceph-mon[66347]: from='client.54820 -' entity='client.iscsi.foo.smithi078.vnplzc' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-09T04:24:26.984 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:26 smithi078 ceph-mon[88406]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:26.985 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:26 smithi078 ceph-mon[83176]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:27.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:26 smithi132 ceph-mon[66347]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:28.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:27 smithi132 ceph-mon[66347]: from='client.54940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:28.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:27 smithi078 ceph-mon[83176]: from='client.54940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:28.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:27 smithi078 ceph-mon[88406]: from='client.54940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:28.320 INFO:teuthology.orchestra.run.smithi078.stdout: "16.2.0", 2024-05-09T04:24:29.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:28 smithi078 ceph-mon[83176]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:24:29.001 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:28 smithi078 ceph-mon[83176]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:29.001 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:28 smithi078 ceph-mon[88406]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:24:29.001 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:28 smithi078 ceph-mon[88406]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:29.025 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-05-09T04:24:29.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:28 smithi132 ceph-mon[66347]: from='mgr.44106 172.21.15.78:0/1341054334' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-09T04:24:29.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:28 smithi132 ceph-mon[66347]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-09T04:24:29.260 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:29 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: ::ffff:172.21.15.132 - - [09/May/2024:04:24:29] "GET /metrics HTTP/1.1" 200 38562 "" "Prometheus/2.43.0" 2024-05-09T04:24:32.078 INFO:teuthology.orchestra.run.smithi078.stdout: "v16.2.2", 2024-05-09T04:24:32.079 INFO:teuthology.orchestra.run.smithi078.stdout: "v16.2.2-20210505", 2024-05-09T04:24:32.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:31 smithi132 ceph-mon[66347]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:32.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:31 smithi132 ceph-mon[66347]: from='client.45003 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:32.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:31 smithi078 ceph-mon[83176]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:32.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:31 smithi078 ceph-mon[83176]: from='client.45003 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:32.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:31 smithi078 ceph-mon[88406]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-09T04:24:32.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:31 smithi078 ceph-mon[88406]: from='client.45003 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-09T04:24:32.778 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-05-09T04:24:32.787 INFO:tasks.cephadm:Teardown begin 2024-05-09T04:24:32.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-09T04:24:32.817 DEBUG:teuthology.orchestra.run.smithi132:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-09T04:24:32.850 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-05-09T04:24:32.850 DEBUG:teuthology.orchestra.run.smithi078:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-09T04:24:32.875 DEBUG:teuthology.orchestra.run.smithi132:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-09T04:24:32.909 INFO:tasks.cephadm:Stopping all daemons... 2024-05-09T04:24:32.909 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-05-09T04:24:32.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.a 2024-05-09T04:24:33.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:32 smithi078 systemd[1]: Stopping Ceph mon.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:24:33.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a[83172]: 2024-05-09T04:24:33.211+0000 7f76bbf01700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:24:33.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a[83172]: 2024-05-09T04:24:33.211+0000 7f76bbf01700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-05-09T04:24:33.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:33 smithi078 podman[130556]: 2024-05-09 04:24:33.314563553 +0000 UTC m=+0.212177362 container died ffed80c876d1e5408ef0005eb5403fa26f98884cf2da7f164ca5849d4f8249c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-05-09T04:24:33.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:24:33] ENGINE Bus STOPPING 2024-05-09T04:24:33.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:24:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-09T04:24:33.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:24:33] ENGINE Bus STOPPED 2024-05-09T04:24:33.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:24:33] ENGINE Bus STARTING 2024-05-09T04:24:33.511 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: CherryPy Checker: 2024-05-09T04:24:33.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: The Application mounted at '' has an empty config. 2024-05-09T04:24:33.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:24:33.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:24:33] ENGINE Serving on http://:::9283 2024-05-09T04:24:33.512 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: [09/May/2024:04:24:33] ENGINE Bus STARTED 2024-05-09T04:24:33.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:33 smithi078 podman[130556]: 2024-05-09 04:24:33.736903997 +0000 UTC m=+0.634517809 container cleanup ffed80c876d1e5408ef0005eb5403fa26f98884cf2da7f164ca5849d4f8249c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240507) 2024-05-09T04:24:33.988 INFO:journalctl@ceph.mon.a.smithi078.stdout:May 09 04:24:33 smithi078 bash[130556]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-a 2024-05-09T04:24:34.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.a.service' 2024-05-09T04:24:34.256 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:24:34.257 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-05-09T04:24:34.257 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-05-09T04:24:34.257 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.c 2024-05-09T04:24:34.510 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:34 smithi078 systemd[1]: Stopping Ceph mon.c for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:24:35.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:34 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c[88402]: 2024-05-09T04:24:34.607+0000 7fc6846a2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:24:35.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:34 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c[88402]: 2024-05-09T04:24:34.607+0000 7fc6846a2700 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-05-09T04:24:35.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:34 smithi078 podman[130690]: 2024-05-09 04:24:34.721408227 +0000 UTC m=+0.242656541 container died f45be4f14efdb450a0af95dc5ae16a010e4d46bacad28ee6279a56bb8c0ca865 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:24:35.317 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:35 smithi078 podman[130690]: 2024-05-09 04:24:35.15042946 +0000 UTC m=+0.671677757 container cleanup f45be4f14efdb450a0af95dc5ae16a010e4d46bacad28ee6279a56bb8c0ca865 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240507, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_BRANCH=HEAD) 2024-05-09T04:24:35.317 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:35 smithi078 bash[130690]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c 2024-05-09T04:24:35.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.c.service' 2024-05-09T04:24:35.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:35 smithi078 podman[130704]: 2024-05-09 04:24:35.318410148 +0000 UTC m=+0.595451285 container remove f45be4f14efdb450a0af95dc5ae16a010e4d46bacad28ee6279a56bb8c0ca865 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-c, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:24:35.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:35 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.c.service: Deactivated successfully. 2024-05-09T04:24:35.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:35 smithi078 systemd[1]: Stopped Ceph mon.c for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:24:35.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:May 09 04:24:35 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.c.service: Consumed 6.537s CPU time. 2024-05-09T04:24:35.598 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:24:35.599 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-05-09T04:24:35.599 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-05-09T04:24:35.599 DEBUG:teuthology.orchestra.run.smithi132:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.b 2024-05-09T04:24:35.921 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:35 smithi132 systemd[1]: Stopping Ceph mon.b for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:24:36.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b[66343]: 2024-05-09T04:24:35.920+0000 7fc751cf9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:24:36.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b[66343]: 2024-05-09T04:24:35.920+0000 7fc751cf9700 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-05-09T04:24:36.253 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:36 smithi132 podman[99820]: 2024-05-09 04:24:36.094829197 +0000 UTC m=+0.299474712 container died d9d8fb1d89f08541cf7511ef2eabf18ac7825d1d3191d965c3abce7b0e534115 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507) 2024-05-09T04:24:36.830 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:36 smithi132 podman[99820]: 2024-05-09 04:24:36.579280188 +0000 UTC m=+0.783925702 container cleanup d9d8fb1d89f08541cf7511ef2eabf18ac7825d1d3191d965c3abce7b0e534115 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.build-date=20240507) 2024-05-09T04:24:36.830 INFO:journalctl@ceph.mon.b.smithi132.stdout:May 09 04:24:36 smithi132 bash[99820]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mon-b 2024-05-09T04:24:36.967 DEBUG:teuthology.orchestra.run.smithi132:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mon.b.service' 2024-05-09T04:24:37.028 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:24:37.028 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-05-09T04:24:37.028 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-05-09T04:24:37.028 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.y 2024-05-09T04:24:37.321 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:37 smithi078 systemd[1]: Stopping Ceph mgr.y for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:24:37.629 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:37 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:24:37.320+0000 7f812812a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:24:37.629 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:37 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y[61741]: 2024-05-09T04:24:37.320+0000 7f812812a700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-05-09T04:24:37.630 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:37 smithi078 podman[130811]: 2024-05-09 04:24:37.362815334 +0000 UTC m=+0.143321654 container died cc8cc8e980a83cf6112186e033829bf618c9e2bf96fd00abcb9f8c1398b0c2c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-05-09T04:24:37.889 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:37 smithi078 podman[130811]: 2024-05-09 04:24:37.704916634 +0000 UTC m=+0.485422941 container cleanup cc8cc8e980a83cf6112186e033829bf618c9e2bf96fd00abcb9f8c1398b0c2c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-05-09T04:24:37.889 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:37 smithi078 bash[130811]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y 2024-05-09T04:24:38.127 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.y.service' 2024-05-09T04:24:38.149 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:37 smithi078 podman[130825]: 2024-05-09 04:24:37.890163276 +0000 UTC m=+0.528461882 container remove cc8cc8e980a83cf6112186e033829bf618c9e2bf96fd00abcb9f8c1398b0c2c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-y, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-05-09T04:24:38.149 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:38 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.y.service: Deactivated successfully. 2024-05-09T04:24:38.149 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:38 smithi078 systemd[1]: Stopped Ceph mgr.y for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:24:38.149 INFO:journalctl@ceph.mgr.y.smithi078.stdout:May 09 04:24:38 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.y.service: Consumed 1min 13.512s CPU time. 2024-05-09T04:24:38.162 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:24:38.162 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-05-09T04:24:38.163 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-05-09T04:24:38.163 DEBUG:teuthology.orchestra.run.smithi132:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.x 2024-05-09T04:24:38.497 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:24:38 smithi132 systemd[1]: Stopping Ceph mgr.x for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:24:38.753 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:24:38 smithi132 podman[99952]: 2024-05-09 04:24:38.498724622 +0000 UTC m=+0.149411113 container died 0569600c45d0e8db14297e5248fe44486118ac690e3477a3cc142f663286d8cc (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-05-09T04:24:39.168 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:24:38 smithi132 podman[99952]: 2024-05-09 04:24:38.859571649 +0000 UTC m=+0.510258135 container cleanup 0569600c45d0e8db14297e5248fe44486118ac690e3477a3cc142f663286d8cc (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-05-09T04:24:39.168 INFO:journalctl@ceph.mgr.x.smithi132.stdout:May 09 04:24:38 smithi132 bash[99952]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-mgr-x 2024-05-09T04:24:39.290 DEBUG:teuthology.orchestra.run.smithi132:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@mgr.x.service' 2024-05-09T04:24:39.327 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:24:39.327 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-05-09T04:24:39.327 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-05-09T04:24:39.327 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.0 2024-05-09T04:24:39.650 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:39 smithi078 systemd[1]: Stopping Ceph osd.0 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:24:40.010 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0[95930]: 2024-05-09T04:24:39.650+0000 7f620909a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:24:40.011 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0[95930]: 2024-05-09T04:24:39.650+0000 7f620909a700 -1 osd.0 160 *** Got signal Terminated *** 2024-05-09T04:24:40.011 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:39 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0[95930]: 2024-05-09T04:24:39.650+0000 7f620909a700 -1 osd.0 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:24:45.293 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:45 smithi078 podman[130930]: 2024-05-09 04:24:45.026901365 +0000 UTC m=+5.511251213 container died ba788dd1e3fe2cb71644c00b04e269085d8b53421890c2d91b7647b9cf99e8e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:24:45.643 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:45 smithi078 podman[130930]: 2024-05-09 04:24:45.384345874 +0000 UTC m=+5.868695721 container cleanup ba788dd1e3fe2cb71644c00b04e269085d8b53421890c2d91b7647b9cf99e8e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:24:45.643 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:45 smithi078 bash[130930]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0 2024-05-09T04:24:46.006 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:45 smithi078 podman[130947]: 2024-05-09 04:24:45.644048946 +0000 UTC m=+0.619895640 container remove ba788dd1e3fe2cb71644c00b04e269085d8b53421890c2d91b7647b9cf99e8e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-05-09T04:24:46.261 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:46 smithi078 podman[131043]: 2024-05-09 04:24:45.907239904 +0000 UTC m=+0.017903852 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:24:46.261 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:46 smithi078 podman[131043]: 2024-05-09 04:24:46.020322461 +0000 UTC m=+0.130986433 container create afe7fdecfb179935b186254b2240c513475c7a324c64222c35bf4a6200159e2c (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240507, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-05-09T04:24:46.558 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:46 smithi078 podman[131043]: 2024-05-09 04:24:46.264718175 +0000 UTC m=+0.375382108 container init afe7fdecfb179935b186254b2240c513475c7a324c64222c35bf4a6200159e2c (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504) 2024-05-09T04:24:46.558 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:46 smithi078 podman[131043]: 2024-05-09 04:24:46.267928098 +0000 UTC m=+0.378592037 container start afe7fdecfb179935b186254b2240c513475c7a324c64222c35bf4a6200159e2c (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240507, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:24:46.558 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:46 smithi078 podman[131043]: 2024-05-09 04:24:46.364873185 +0000 UTC m=+0.475537116 container attach afe7fdecfb179935b186254b2240c513475c7a324c64222c35bf4a6200159e2c (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-deactivate, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-05-09T04:24:46.900 INFO:journalctl@ceph.osd.0.smithi078.stdout:May 09 04:24:46 smithi078 podman[131043]: 2024-05-09 04:24:46.55835837 +0000 UTC m=+0.669022294 container died afe7fdecfb179935b186254b2240c513475c7a324c64222c35bf4a6200159e2c (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240507, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0) 2024-05-09T04:24:47.174 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.0.service' 2024-05-09T04:24:47.219 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:24:47.219 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-05-09T04:24:47.219 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-05-09T04:24:47.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.1 2024-05-09T04:24:47.413 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:47 smithi078 systemd[1]: Stopping Ceph osd.1 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:24:47.761 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:47 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1[101122]: 2024-05-09T04:24:47.581+0000 7f59ceeec700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:24:47.761 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:47 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1[101122]: 2024-05-09T04:24:47.581+0000 7f59ceeec700 -1 osd.1 160 *** Got signal Terminated *** 2024-05-09T04:24:47.761 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:47 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1[101122]: 2024-05-09T04:24:47.581+0000 7f59ceeec700 -1 osd.1 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:24:53.314 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:53 smithi078 podman[131149]: 2024-05-09 04:24:53.037700068 +0000 UTC m=+5.579617537 container died c1e61031cb25a21eef32e2e15dd1b45140ac70deb478435d14c7297c8ad7242d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.build-date=20240507, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-05-09T04:24:53.655 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:53 smithi078 podman[131149]: 2024-05-09 04:24:53.413300103 +0000 UTC m=+5.955217579 container cleanup c1e61031cb25a21eef32e2e15dd1b45140ac70deb478435d14c7297c8ad7242d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:24:53.655 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:53 smithi078 bash[131149]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1 2024-05-09T04:24:53.985 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:53 smithi078 podman[131176]: 2024-05-09 04:24:53.656612067 +0000 UTC m=+0.618972170 container remove c1e61031cb25a21eef32e2e15dd1b45140ac70deb478435d14c7297c8ad7242d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-05-09T04:24:54.242 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:53 smithi078 podman[131270]: 2024-05-09 04:24:53.889800619 +0000 UTC m=+0.018309295 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:24:54.243 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:53 smithi078 podman[131270]: 2024-05-09 04:24:53.999364034 +0000 UTC m=+0.127872740 container create 5aa75e509e542cfc3dce8c70627d3e84347f4ec4ce255d21b6d95d08aaa02ac3 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-05-09T04:24:54.243 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:54 smithi078 podman[131270]: 2024-05-09 04:24:54.24382754 +0000 UTC m=+0.372336214 container init 5aa75e509e542cfc3dce8c70627d3e84347f4ec4ce255d21b6d95d08aaa02ac3 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:24:54.510 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:54 smithi078 podman[131270]: 2024-05-09 04:24:54.247157737 +0000 UTC m=+0.375666410 container start 5aa75e509e542cfc3dce8c70627d3e84347f4ec4ce255d21b6d95d08aaa02ac3 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240507, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True) 2024-05-09T04:24:54.511 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:54 smithi078 podman[131270]: 2024-05-09 04:24:54.327090287 +0000 UTC m=+0.455598956 container attach 5aa75e509e542cfc3dce8c70627d3e84347f4ec4ce255d21b6d95d08aaa02ac3 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-05-09T04:24:54.820 INFO:journalctl@ceph.osd.1.smithi078.stdout:May 09 04:24:54 smithi078 podman[131270]: 2024-05-09 04:24:54.539611483 +0000 UTC m=+0.668120158 container died 5aa75e509e542cfc3dce8c70627d3e84347f4ec4ce255d21b6d95d08aaa02ac3 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, GIT_CLEAN=True) 2024-05-09T04:24:55.123 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.1.service' 2024-05-09T04:24:55.163 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:24:55.163 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-05-09T04:24:55.163 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-05-09T04:24:55.163 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.2 2024-05-09T04:24:55.500 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:24:55 smithi078 systemd[1]: Stopping Ceph osd.2 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:24:55.761 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:24:55 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2[107595]: 2024-05-09T04:24:55.500+0000 7ff805a63700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:24:55.761 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:24:55 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2[107595]: 2024-05-09T04:24:55.500+0000 7ff805a63700 -1 osd.2 160 *** Got signal Terminated *** 2024-05-09T04:24:55.761 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:24:55 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2[107595]: 2024-05-09T04:24:55.500+0000 7ff805a63700 -1 osd.2 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:25:01.132 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:25:00 smithi078 podman[131377]: 2024-05-09 04:25:00.824109565 +0000 UTC m=+5.435961862 container died 65617f5ef4aeaa461e48c86505e96b46c79fe18e284dc524d8bab0c4f3104483 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-05-09T04:25:01.132 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:25:01 smithi078 podman[131377]: 2024-05-09 04:25:01.133503791 +0000 UTC m=+5.745356063 container cleanup 65617f5ef4aeaa461e48c86505e96b46c79fe18e284dc524d8bab0c4f3104483 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2, org.label-schema.build-date=20240507, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-05-09T04:25:01.510 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:25:01 smithi078 bash[131377]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2 2024-05-09T04:25:01.511 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:25:01 smithi078 podman[131392]: 2024-05-09 04:25:01.351496679 +0000 UTC m=+0.522451359 container remove 65617f5ef4aeaa461e48c86505e96b46c79fe18e284dc524d8bab0c4f3104483 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2, ceph=True, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504) 2024-05-09T04:25:02.011 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:25:01 smithi078 podman[131485]: 2024-05-09 04:25:01.760873975 +0000 UTC m=+0.111345000 container create 4f58e36bab0a49b0ab62b95540a6f9a374c41c53820196d087cc6a892739d567 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:25:02.011 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:25:01 smithi078 podman[131485]: 2024-05-09 04:25:01.667508286 +0000 UTC m=+0.017979324 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:25:02.011 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:25:01 smithi078 podman[131485]: 2024-05-09 04:25:01.963774012 +0000 UTC m=+0.314245037 container init 4f58e36bab0a49b0ab62b95540a6f9a374c41c53820196d087cc6a892739d567 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-05-09T04:25:02.011 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:25:01 smithi078 podman[131485]: 2024-05-09 04:25:01.967159458 +0000 UTC m=+0.317630487 container start 4f58e36bab0a49b0ab62b95540a6f9a374c41c53820196d087cc6a892739d567 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-05-09T04:25:02.511 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:25:02 smithi078 podman[131485]: 2024-05-09 04:25:02.047053986 +0000 UTC m=+0.397525021 container attach 4f58e36bab0a49b0ab62b95540a6f9a374c41c53820196d087cc6a892739d567 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:25:02.511 INFO:journalctl@ceph.osd.2.smithi078.stdout:May 09 04:25:02 smithi078 podman[131485]: 2024-05-09 04:25:02.259372053 +0000 UTC m=+0.609843078 container died 4f58e36bab0a49b0ab62b95540a6f9a374c41c53820196d087cc6a892739d567 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-05-09T04:25:02.899 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.2.service' 2024-05-09T04:25:02.939 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:25:02.939 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-05-09T04:25:02.939 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-05-09T04:25:02.939 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.3 2024-05-09T04:25:03.260 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:03 smithi078 systemd[1]: Stopping Ceph osd.3 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:25:03.554 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3[114204]: 2024-05-09T04:25:03.285+0000 7fb699b1e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:25:03.554 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3[114204]: 2024-05-09T04:25:03.285+0000 7fb699b1e700 -1 osd.3 160 *** Got signal Terminated *** 2024-05-09T04:25:03.554 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3[114204]: 2024-05-09T04:25:03.285+0000 7fb699b1e700 -1 osd.3 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:25:04.011 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:03.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi078.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.78:8443: connect: connection refused" 2024-05-09T04:25:04.011 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:03 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:03.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi132.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.132:8443: connect: connection refused" 2024-05-09T04:25:06.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:05 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:05.504+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:06.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:06.475+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:07.170 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:06.893Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.78:8765: connect: connection refused" 2024-05-09T04:25:07.170 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:06.898Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.78:8765: connect: connection refused" 2024-05-09T04:25:07.170 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:06.898Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.78:8765: connect: connection refused" 2024-05-09T04:25:07.170 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:06.898Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.78:8765: connect: connection refused" 2024-05-09T04:25:07.170 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:06 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:06.898Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.78:8765: connect: connection refused" 2024-05-09T04:25:07.820 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:07 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:07.518+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:08.252 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:07 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:07.820+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:08.807 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:08 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:08.539+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:09.011 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:08 smithi078 podman[131591]: 2024-05-09 04:25:08.746940701 +0000 UTC m=+5.585512246 container died cfe832ae10c36d33f5465f876e952857ef7414cb58eb8684c3a0ac8ee5c3073d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240507, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:25:09.182 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:08 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:08.807+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:09.387 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:09 smithi078 podman[131591]: 2024-05-09 04:25:09.103296503 +0000 UTC m=+5.941868056 container cleanup cfe832ae10c36d33f5465f876e952857ef7414cb58eb8684c3a0ac8ee5c3073d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:25:09.387 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:09 smithi078 bash[131591]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3 2024-05-09T04:25:09.503 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:09 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:25:09.182+0000 7f3b33dda700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.906995+0000 front 2024-05-09T04:24:42.906972+0000 (oldest deadline 2024-05-09T04:25:08.807002+0000) 2024-05-09T04:25:09.724 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:09 smithi078 podman[131608]: 2024-05-09 04:25:09.388353501 +0000 UTC m=+0.632266014 container remove cfe832ae10c36d33f5465f876e952857ef7414cb58eb8684c3a0ac8ee5c3073d (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:25:10.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:09 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:09.764+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:10.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:09 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:09.546+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:10.003 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:09 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:09.781+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:10.007 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:09 smithi078 podman[131701]: 2024-05-09 04:25:09.628319218 +0000 UTC m=+0.018234100 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:25:10.007 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:09 smithi078 podman[131701]: 2024-05-09 04:25:09.7309878 +0000 UTC m=+0.120902707 container create e4bd03a73109d741539527834a2a81894b81828fb1a4e6612ca2f77937e8cc44 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD) 2024-05-09T04:25:10.008 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:09 smithi078 podman[131701]: 2024-05-09 04:25:09.925178332 +0000 UTC m=+0.315093198 container init e4bd03a73109d741539527834a2a81894b81828fb1a4e6612ca2f77937e8cc44 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-05-09T04:25:10.008 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:09 smithi078 podman[131701]: 2024-05-09 04:25:09.92841191 +0000 UTC m=+0.318326776 container start e4bd03a73109d741539527834a2a81894b81828fb1a4e6612ca2f77937e8cc44 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-05-09T04:25:10.008 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:10 smithi078 podman[131701]: 2024-05-09 04:25:10.008570261 +0000 UTC m=+0.398485126 container attach e4bd03a73109d741539527834a2a81894b81828fb1a4e6612ca2f77937e8cc44 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True) 2024-05-09T04:25:10.260 INFO:journalctl@ceph.osd.3.smithi078.stdout:May 09 04:25:10 smithi078 podman[131701]: 2024-05-09 04:25:10.224664406 +0000 UTC m=+0.614579273 container died e4bd03a73109d741539527834a2a81894b81828fb1a4e6612ca2f77937e8cc44 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-3-deactivate, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.build-date=20240507, CEPH_POINT_RELEASE=) 2024-05-09T04:25:10.500 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:10 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:25:10.157+0000 7f3b33dda700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.906995+0000 front 2024-05-09T04:24:42.906972+0000 (oldest deadline 2024-05-09T04:25:08.807002+0000) 2024-05-09T04:25:10.753 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:10 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:10.726+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:10.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:10 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:10.499+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:10.803 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.3.service' 2024-05-09T04:25:10.844 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:25:10.845 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-05-09T04:25:10.845 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-05-09T04:25:10.845 DEBUG:teuthology.orchestra.run.smithi132:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.4 2024-05-09T04:25:11.153 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:10 smithi132 systemd[1]: Stopping Ceph osd.4 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:25:11.153 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:10 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:10.753+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:11.503 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:11 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:25:11.153+0000 7f3b377e6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:25:11.503 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:11 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:25:11.153+0000 7f3b377e6700 -1 osd.4 160 *** Got signal Terminated *** 2024-05-09T04:25:11.503 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:11 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:25:11.153+0000 7f3b377e6700 -1 osd.4 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:25:11.503 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:11 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:25:11.199+0000 7f3b33dda700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.906995+0000 front 2024-05-09T04:24:42.906972+0000 (oldest deadline 2024-05-09T04:25:08.807002+0000) 2024-05-09T04:25:12.002 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:11 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:11.769+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:12.003 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:11 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:11.734+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:12.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:11 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:11.533+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:12.502 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:12 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:25:12.217+0000 7f3b33dda700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.906995+0000 front 2024-05-09T04:24:42.906972+0000 (oldest deadline 2024-05-09T04:25:08.807002+0000) 2024-05-09T04:25:13.003 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:12 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:12.694+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:13.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:12 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:12.789+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:13.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:12 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:12.581+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:13.502 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:25:13.188+0000 7f3b33dda700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.906995+0000 front 2024-05-09T04:24:42.906972+0000 (oldest deadline 2024-05-09T04:25:08.807002+0000) 2024-05-09T04:25:14.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:13.588+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:14.003 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:13.657+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:14.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:13.799+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:14.010 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:13.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi078.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.78:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi132.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.132:8443: connect: connection refused" 2024-05-09T04:25:14.011 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:13.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi078.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.78:8443: connect: connection refused" 2024-05-09T04:25:14.011 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:13 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:13.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi132.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.132:8443: connect: connection refused" 2024-05-09T04:25:14.502 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:25:14.230+0000 7f3b33dda700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.906995+0000 front 2024-05-09T04:24:42.906972+0000 (oldest deadline 2024-05-09T04:25:08.807002+0000) 2024-05-09T04:25:14.503 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:25:14.230+0000 7f3b33dda700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:48.807659+0000 front 2024-05-09T04:24:48.807674+0000 (oldest deadline 2024-05-09T04:25:14.107503+0000) 2024-05-09T04:25:14.821 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:14.615+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:14.821 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:14.542+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:15.236 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:14 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:14.821+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:15.502 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:25:15.236+0000 7f3b33dda700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.906995+0000 front 2024-05-09T04:24:42.906972+0000 (oldest deadline 2024-05-09T04:25:08.807002+0000) 2024-05-09T04:25:15.503 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4[76941]: 2024-05-09T04:25:15.236+0000 7f3b33dda700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:48.807659+0000 front 2024-05-09T04:24:48.807674+0000 (oldest deadline 2024-05-09T04:25:14.107503+0000) 2024-05-09T04:25:15.836 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:15.569+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:15.837 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:15.577+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:16.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:15.837+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:16.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:15 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:15.837+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:16.794 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:16 smithi132 podman[100086]: 2024-05-09 04:25:16.526408619 +0000 UTC m=+5.482761603 container died cdf9ea05dcb05084f432ba81f7a0901a5577e8672bb7bfaeda80684901b67f7c (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4, org.label-schema.build-date=20240507, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_BRANCH=HEAD) 2024-05-09T04:25:16.795 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:16 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:16.575+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:16.795 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:16 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:16.578+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:16.795 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:16 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:16.578+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:17.146 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:16 smithi132 podman[100086]: 2024-05-09 04:25:16.962098712 +0000 UTC m=+5.918451675 container cleanup cdf9ea05dcb05084f432ba81f7a0901a5577e8672bb7bfaeda80684901b67f7c (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-05-09T04:25:17.146 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:16 smithi132 bash[100086]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4 2024-05-09T04:25:17.146 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:16 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:16.853+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:17.146 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:16 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:16.853+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:17.502 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:17 smithi132 podman[100104]: 2024-05-09 04:25:17.147304551 +0000 UTC m=+0.616325558 container remove cdf9ea05dcb05084f432ba81f7a0901a5577e8672bb7bfaeda80684901b67f7c (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-05-09T04:25:17.858 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:17.554+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:17.858 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:17.554+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.401333+0000 front 2024-05-09T04:24:51.401323+0000 (oldest deadline 2024-05-09T04:25:17.301250+0000) 2024-05-09T04:25:17.858 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:17.613+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:17.858 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:17.613+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:17.859 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:17 smithi132 podman[100200]: 2024-05-09 04:25:17.488054986 +0000 UTC m=+0.018086450 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:25:17.859 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:17 smithi132 podman[100200]: 2024-05-09 04:25:17.62309305 +0000 UTC m=+0.153124508 container create 39ceb35117bfc775cdf3b4aaa53838c5d54036e5e219d8189d36bc21014ba0c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504) 2024-05-09T04:25:17.859 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:17 smithi132 podman[100200]: 2024-05-09 04:25:17.859018817 +0000 UTC m=+0.389050274 container init 39ceb35117bfc775cdf3b4aaa53838c5d54036e5e219d8189d36bc21014ba0c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-deactivate, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, GIT_BRANCH=HEAD) 2024-05-09T04:25:18.166 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:17.884+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:18.166 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:17 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:17.884+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:18.167 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:17 smithi132 podman[100200]: 2024-05-09 04:25:17.862709302 +0000 UTC m=+0.392740770 container start 39ceb35117bfc775cdf3b4aaa53838c5d54036e5e219d8189d36bc21014ba0c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:25:18.167 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:18 smithi132 podman[100200]: 2024-05-09 04:25:18.000817104 +0000 UTC m=+0.530848582 container attach 39ceb35117bfc775cdf3b4aaa53838c5d54036e5e219d8189d36bc21014ba0c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-05-09T04:25:18.167 INFO:journalctl@ceph.osd.4.smithi132.stdout:May 09 04:25:18 smithi132 podman[100200]: 2024-05-09 04:25:18.167594003 +0000 UTC m=+0.697625461 container died 39ceb35117bfc775cdf3b4aaa53838c5d54036e5e219d8189d36bc21014ba0c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:25:18.753 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:18.585+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:18.753 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:18.585+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.401333+0000 front 2024-05-09T04:24:51.401323+0000 (oldest deadline 2024-05-09T04:25:17.301250+0000) 2024-05-09T04:25:18.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:18.584+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:18.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:18.584+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:18.867 DEBUG:teuthology.orchestra.run.smithi132:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.4.service' 2024-05-09T04:25:18.911 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:25:18.911 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-05-09T04:25:18.911 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-05-09T04:25:18.911 DEBUG:teuthology.orchestra.run.smithi132:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.5 2024-05-09T04:25:19.232 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:18.915+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:19.232 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:18 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:18.915+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:19.233 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:18 smithi132 systemd[1]: Stopping Ceph osd.5 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:25:19.233 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:19.232+0000 7fd2b2fe7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:25:19.233 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:19.232+0000 7fd2b2fe7700 -1 osd.5 160 *** Got signal Terminated *** 2024-05-09T04:25:19.233 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:19.232+0000 7fd2b2fe7700 -1 osd.5 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:25:19.959 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:19.579+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:19.959 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:19.579+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.401333+0000 front 2024-05-09T04:24:51.401323+0000 (oldest deadline 2024-05-09T04:25:17.301250+0000) 2024-05-09T04:25:19.959 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:19.551+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:19.959 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:19.551+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:20.252 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:19.959+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:20.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:19 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:19.959+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:20.940 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:20.584+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:20.940 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:20.584+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.401333+0000 front 2024-05-09T04:24:51.401323+0000 (oldest deadline 2024-05-09T04:25:17.301250+0000) 2024-05-09T04:25:20.941 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:20.540+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:20.941 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:20.540+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:21.252 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:20.940+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:21.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:20 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:20.940+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:21.957 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:21 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:21.633+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:21.957 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:21 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:21.633+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.401333+0000 front 2024-05-09T04:24:51.401323+0000 (oldest deadline 2024-05-09T04:25:17.301250+0000) 2024-05-09T04:25:21.957 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:21 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:21.543+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:21.957 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:21 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:21.543+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:22.252 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:21 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:21.956+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:22.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:21 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:21.956+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:22.941 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:22 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:22.606+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:22.942 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:22 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:22.606+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.401333+0000 front 2024-05-09T04:24:51.401323+0000 (oldest deadline 2024-05-09T04:25:17.301250+0000) 2024-05-09T04:25:22.942 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:22 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:22.562+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:22.942 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:22 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:22.562+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:23.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:22 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:22.942+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:23.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:22 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:22.942+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:23.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:22 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:22.942+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:56.994763+0000 front 2024-05-09T04:24:56.994669+0000 (oldest deadline 2024-05-09T04:25:22.294593+0000) 2024-05-09T04:25:23.894 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:23.649+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:43.800375+0000 front 2024-05-09T04:24:43.800390+0000 (oldest deadline 2024-05-09T04:25:09.700341+0000) 2024-05-09T04:25:23.894 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5[81463]: 2024-05-09T04:25:23.649+0000 7fd2ab5d3700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.401333+0000 front 2024-05-09T04:24:51.401323+0000 (oldest deadline 2024-05-09T04:25:17.301250+0000) 2024-05-09T04:25:23.894 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:23.611+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:23.895 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:23.611+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:23.895 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:23.611+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:23.922 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:23.556Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi132.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.132:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi078.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.78:8443: connect: connection refused" 2024-05-09T04:25:23.922 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:23.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi078.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.78:8443: connect: connection refused" 2024-05-09T04:25:23.922 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:23 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:23.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi132.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.132:8443: connect: connection refused" 2024-05-09T04:25:24.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:23.894+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:24.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:23.894+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:24.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:23 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:23.894+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:56.994763+0000 front 2024-05-09T04:24:56.994669+0000 (oldest deadline 2024-05-09T04:25:22.294593+0000) 2024-05-09T04:25:24.933 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:24 smithi132 podman[100305]: 2024-05-09 04:25:24.73678409 +0000 UTC m=+5.602366591 container died 408a908ef3cc5287f942c38c9fff401b0c24f2a3c4baee3c768d94d25966e85e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240507, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-05-09T04:25:24.933 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:24.579+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:24.933 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:24.579+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:24.933 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:24.579+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:25.184 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:25 smithi132 podman[100305]: 2024-05-09 04:25:25.182809028 +0000 UTC m=+6.048391523 container cleanup 408a908ef3cc5287f942c38c9fff401b0c24f2a3c4baee3c768d94d25966e85e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-05-09T04:25:25.184 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:25 smithi132 bash[100305]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5 2024-05-09T04:25:25.184 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:24.933+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:25.184 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:24.933+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:25.184 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:24 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:24.933+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:56.994763+0000 front 2024-05-09T04:24:56.994669+0000 (oldest deadline 2024-05-09T04:25:22.294593+0000) 2024-05-09T04:25:25.753 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:25 smithi132 podman[100322]: 2024-05-09 04:25:25.484448333 +0000 UTC m=+0.748956854 container remove 408a908ef3cc5287f942c38c9fff401b0c24f2a3c4baee3c768d94d25966e85e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-05-09T04:25:25.754 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:25.568+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:25.754 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:25.568+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:25.754 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:25.568+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:26.096 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:25 smithi132 podman[100415]: 2024-05-09 04:25:25.713037566 +0000 UTC m=+0.018210787 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:25:26.096 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:25 smithi132 podman[100415]: 2024-05-09 04:25:25.873178001 +0000 UTC m=+0.178351258 container create 55dcd0ab10973a46d8ebaab77dde0abe4c8f5d74aa5cc1a1bb13aae73b970a6e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:25:26.096 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:25.958+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:26.097 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:25.958+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:26.097 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:25 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:25.958+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:56.994763+0000 front 2024-05-09T04:24:56.994669+0000 (oldest deadline 2024-05-09T04:25:22.294593+0000) 2024-05-09T04:25:26.391 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:26 smithi132 podman[100415]: 2024-05-09 04:25:26.097310924 +0000 UTC m=+0.402484139 container init 55dcd0ab10973a46d8ebaab77dde0abe4c8f5d74aa5cc1a1bb13aae73b970a6e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:25:26.391 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:26 smithi132 podman[100415]: 2024-05-09 04:25:26.101060289 +0000 UTC m=+0.406233511 container start 55dcd0ab10973a46d8ebaab77dde0abe4c8f5d74aa5cc1a1bb13aae73b970a6e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504) 2024-05-09T04:25:26.391 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:26 smithi132 podman[100415]: 2024-05-09 04:25:26.171471888 +0000 UTC m=+0.476645111 container attach 55dcd0ab10973a46d8ebaab77dde0abe4c8f5d74aa5cc1a1bb13aae73b970a6e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, ceph=True) 2024-05-09T04:25:26.690 INFO:journalctl@ceph.osd.5.smithi132.stdout:May 09 04:25:26 smithi132 podman[100415]: 2024-05-09 04:25:26.392573847 +0000 UTC m=+0.697747066 container died 55dcd0ab10973a46d8ebaab77dde0abe4c8f5d74aa5cc1a1bb13aae73b970a6e (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240507, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-05-09T04:25:26.691 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:26 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:26.588+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:26.691 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:26 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:26.588+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:26.691 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:26 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:26.588+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:26.971 DEBUG:teuthology.orchestra.run.smithi132:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.5.service' 2024-05-09T04:25:27.011 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:25:27.011 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-05-09T04:25:27.012 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-05-09T04:25:27.012 DEBUG:teuthology.orchestra.run.smithi132:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.6 2024-05-09T04:25:27.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:26 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:26.959+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:27.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:26 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:26.959+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:27.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:26 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:26.959+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:56.994763+0000 front 2024-05-09T04:24:56.994669+0000 (oldest deadline 2024-05-09T04:25:22.294593+0000) 2024-05-09T04:25:27.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:27 smithi132 systemd[1]: Stopping Ceph osd.6 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:25:27.603 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:27.352+0000 7f1d10e10700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:25:27.604 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:27.352+0000 7f1d10e10700 -1 osd.6 160 *** Got signal Terminated *** 2024-05-09T04:25:27.604 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:27.352+0000 7f1d10e10700 -1 osd.6 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:25:27.920 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:27.604+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:27.920 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:27.604+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:27.920 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:27.604+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:28.252 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:27.921+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:28.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:27.921+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:28.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:27 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:27.921+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:56.994763+0000 front 2024-05-09T04:24:56.994669+0000 (oldest deadline 2024-05-09T04:25:22.294593+0000) 2024-05-09T04:25:29.002 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:28.886+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:29.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:28.886+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:29.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:28.886+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:56.994763+0000 front 2024-05-09T04:24:56.994669+0000 (oldest deadline 2024-05-09T04:25:22.294593+0000) 2024-05-09T04:25:29.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:28.645+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:29.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:28.645+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:29.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:28 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:28.646+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:30.002 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:29 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:29.843+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:30.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:29 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:29.843+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:30.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:29 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:29.843+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:56.994763+0000 front 2024-05-09T04:24:56.994669+0000 (oldest deadline 2024-05-09T04:25:22.294593+0000) 2024-05-09T04:25:30.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:29 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:29.671+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:30.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:29 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:29.671+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:30.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:29 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:29.671+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:30.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:29 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:29.671+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:07.370022+0000 front 2024-05-09T04:25:07.369986+0000 (oldest deadline 2024-05-09T04:25:29.069841+0000) 2024-05-09T04:25:31.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:30.819+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:31.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:30.819+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:31.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:30.819+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:56.994763+0000 front 2024-05-09T04:24:56.994669+0000 (oldest deadline 2024-05-09T04:25:22.294593+0000) 2024-05-09T04:25:31.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:30.819+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:05.795473+0000 front 2024-05-09T04:25:05.795528+0000 (oldest deadline 2024-05-09T04:25:30.495514+0000) 2024-05-09T04:25:31.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:30.664+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:31.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:30.664+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:31.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:30.664+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:31.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:30 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:30.664+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:07.370022+0000 front 2024-05-09T04:25:07.369986+0000 (oldest deadline 2024-05-09T04:25:29.069841+0000) 2024-05-09T04:25:32.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:31 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:31.853+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:38.991597+0000 front 2024-05-09T04:24:38.991760+0000 (oldest deadline 2024-05-09T04:25:07.792164+0000) 2024-05-09T04:25:32.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:31 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:31.853+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:51.193915+0000 front 2024-05-09T04:24:51.193967+0000 (oldest deadline 2024-05-09T04:25:15.293817+0000) 2024-05-09T04:25:32.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:31 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:31.853+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:56.994763+0000 front 2024-05-09T04:24:56.994669+0000 (oldest deadline 2024-05-09T04:25:22.294593+0000) 2024-05-09T04:25:32.003 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:31 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6[86277]: 2024-05-09T04:25:31.853+0000 7f1d0d404700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:05.795473+0000 front 2024-05-09T04:25:05.795528+0000 (oldest deadline 2024-05-09T04:25:30.495514+0000) 2024-05-09T04:25:32.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:31 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:31.634+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:32.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:31 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:31.634+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:32.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:31 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:31.634+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:32.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:31 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:31.634+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:07.370022+0000 front 2024-05-09T04:25:07.369986+0000 (oldest deadline 2024-05-09T04:25:29.069841+0000) 2024-05-09T04:25:32.970 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:32 smithi132 podman[100519]: 2024-05-09 04:25:32.679141885 +0000 UTC m=+5.434017889 container died 01f14aea01f3e351ffb9ea3d84ff4e9e0a20d14e47d6890ba5512200abca1336 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:25:32.971 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:32 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:32.588+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:32.971 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:32 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:32.588+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:32.971 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:32 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:32.588+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:32.971 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:32 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:32.588+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:07.370022+0000 front 2024-05-09T04:25:07.369986+0000 (oldest deadline 2024-05-09T04:25:29.069841+0000) 2024-05-09T04:25:33.252 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:33 smithi132 podman[100519]: 2024-05-09 04:25:33.136454465 +0000 UTC m=+5.891328795 container cleanup 01f14aea01f3e351ffb9ea3d84ff4e9e0a20d14e47d6890ba5512200abca1336 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, org.label-schema.license=GPLv2, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-09T04:25:33.253 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:33 smithi132 bash[100519]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6 2024-05-09T04:25:33.753 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:33 smithi132 podman[100536]: 2024-05-09 04:25:33.429665339 +0000 UTC m=+0.743881146 container remove 01f14aea01f3e351ffb9ea3d84ff4e9e0a20d14e47d6890ba5512200abca1336 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240507, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-05-09T04:25:33.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:33.595+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:33.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:33.595+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:33.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:33.595+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:33.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:33 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:33.595+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:07.370022+0000 front 2024-05-09T04:25:07.369986+0000 (oldest deadline 2024-05-09T04:25:29.069841+0000) 2024-05-09T04:25:34.011 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:33.556Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi078.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.78:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi132.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.132:8443: connect: connection refused" 2024-05-09T04:25:34.011 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:33.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi078.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.78:8443: connect: connection refused" 2024-05-09T04:25:34.012 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:33 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:33.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi132.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.132:8443: connect: connection refused" 2024-05-09T04:25:34.015 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:33 smithi132 podman[100633]: 2024-05-09 04:25:33.763895353 +0000 UTC m=+0.109728870 container create 06eba2214402d1591dcb05da88404e845a94f9ea37ebe16e768ca8a45f898b30 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240507) 2024-05-09T04:25:34.015 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:33 smithi132 podman[100633]: 2024-05-09 04:25:33.671935696 +0000 UTC m=+0.017769228 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:25:34.316 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:34 smithi132 podman[100633]: 2024-05-09 04:25:34.016743002 +0000 UTC m=+0.362576527 container init 06eba2214402d1591dcb05da88404e845a94f9ea37ebe16e768ca8a45f898b30 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, org.label-schema.build-date=20240507, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux ) 2024-05-09T04:25:34.316 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:34 smithi132 podman[100633]: 2024-05-09 04:25:34.020248436 +0000 UTC m=+0.366081954 container start 06eba2214402d1591dcb05da88404e845a94f9ea37ebe16e768ca8a45f898b30 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.build-date=20240507, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504) 2024-05-09T04:25:34.317 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:34 smithi132 podman[100633]: 2024-05-09 04:25:34.099586914 +0000 UTC m=+0.445420440 container attach 06eba2214402d1591dcb05da88404e845a94f9ea37ebe16e768ca8a45f898b30 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240507) 2024-05-09T04:25:34.677 INFO:journalctl@ceph.osd.6.smithi132.stdout:May 09 04:25:34 smithi132 podman[100633]: 2024-05-09 04:25:34.317387787 +0000 UTC m=+0.663221319 container died 06eba2214402d1591dcb05da88404e845a94f9ea37ebe16e768ca8a45f898b30 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-6-deactivate, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240507, org.label-schema.schema-version=1.0, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True) 2024-05-09T04:25:34.677 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:34.562+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:34.677 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:34.562+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:34.677 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:34.562+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:34.677 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:34 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:34.562+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:07.370022+0000 front 2024-05-09T04:25:07.369986+0000 (oldest deadline 2024-05-09T04:25:29.069841+0000) 2024-05-09T04:25:35.063 DEBUG:teuthology.orchestra.run.smithi132:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.6.service' 2024-05-09T04:25:35.103 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:25:35.104 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-05-09T04:25:35.104 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-05-09T04:25:35.104 DEBUG:teuthology.orchestra.run.smithi132:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.7 2024-05-09T04:25:35.439 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:35 smithi132 systemd[1]: Stopping Ceph osd.7 for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:25:35.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:35.440+0000 7f92a9887700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-09T04:25:35.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:35.440+0000 7f92a9887700 -1 osd.7 160 *** Got signal Terminated *** 2024-05-09T04:25:35.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:35.440+0000 7f92a9887700 -1 osd.7 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-09T04:25:35.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:35.549+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:35.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:35.549+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:35.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:35.549+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:35.753 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:35 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:35.549+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:07.370022+0000 front 2024-05-09T04:25:07.369986+0000 (oldest deadline 2024-05-09T04:25:29.069841+0000) 2024-05-09T04:25:37.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:36.575+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:37.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:36.575+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:37.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:36.575+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:37.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:36 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:36.575+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:07.370022+0000 front 2024-05-09T04:25:07.369986+0000 (oldest deadline 2024-05-09T04:25:29.069841+0000) 2024-05-09T04:25:37.899 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:37.564+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:37.899 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:37.564+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:37.899 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:37.564+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:37.899 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:37 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:37.564+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:07.370022+0000 front 2024-05-09T04:25:07.369986+0000 (oldest deadline 2024-05-09T04:25:29.069841+0000) 2024-05-09T04:25:38.810 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:38.523+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:38.810 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:38.523+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:38.810 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:38.523+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:38.810 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:38.523+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:07.370022+0000 front 2024-05-09T04:25:07.369986+0000 (oldest deadline 2024-05-09T04:25:29.069841+0000) 2024-05-09T04:25:38.810 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:38 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:38.523+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.132:6804 osd.4 since back 2024-05-09T04:25:13.770790+0000 front 2024-05-09T04:25:13.770715+0000 (oldest deadline 2024-05-09T04:25:37.870685+0000) 2024-05-09T04:25:40.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:39.532+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-05-09T04:24:42.866898+0000 front 2024-05-09T04:24:42.866956+0000 (oldest deadline 2024-05-09T04:25:05.166710+0000) 2024-05-09T04:25:40.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:39.532+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-05-09T04:24:50.968341+0000 front 2024-05-09T04:24:50.968231+0000 (oldest deadline 2024-05-09T04:25:15.668123+0000) 2024-05-09T04:25:40.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:39.532+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-05-09T04:24:59.769127+0000 front 2024-05-09T04:24:59.769168+0000 (oldest deadline 2024-05-09T04:25:23.269088+0000) 2024-05-09T04:25:40.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:39.532+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-05-09T04:25:07.370022+0000 front 2024-05-09T04:25:07.369986+0000 (oldest deadline 2024-05-09T04:25:29.069841+0000) 2024-05-09T04:25:40.003 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:39 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7[90815]: 2024-05-09T04:25:39.532+0000 7f92a1e73700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.132:6804 osd.4 since back 2024-05-09T04:25:13.770790+0000 front 2024-05-09T04:25:13.770715+0000 (oldest deadline 2024-05-09T04:25:37.870685+0000) 2024-05-09T04:25:41.115 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:40 smithi132 podman[100738]: 2024-05-09 04:25:40.844336675 +0000 UTC m=+5.515321762 container died f50fb4f7e63c87b77ddfb08143a8f69a0ff8f3de39556a1bdca8bc71076c47a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553) 2024-05-09T04:25:41.472 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:41 smithi132 podman[100738]: 2024-05-09 04:25:41.197524649 +0000 UTC m=+5.868509735 container cleanup f50fb4f7e63c87b77ddfb08143a8f69a0ff8f3de39556a1bdca8bc71076c47a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7, io.buildah.version=1.33.5, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:25:41.473 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:41 smithi132 bash[100738]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7 2024-05-09T04:25:41.752 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:41 smithi132 podman[100767]: 2024-05-09 04:25:41.473774852 +0000 UTC m=+0.624404809 container remove f50fb4f7e63c87b77ddfb08143a8f69a0ff8f3de39556a1bdca8bc71076c47a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240507, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504) 2024-05-09T04:25:42.085 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:41 smithi132 podman[100862]: 2024-05-09 04:25:41.72742075 +0000 UTC m=+0.018302507 image pull af4a4df59e0ae3833b4409b12cb7db15106e9e87264b888efc4bb545bee0fce2 quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082 2024-05-09T04:25:42.085 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:41 smithi132 podman[100862]: 2024-05-09 04:25:41.841468608 +0000 UTC m=+0.132350349 container create 33407b9042d9f03e629cc22595faf0071262daa53e4774b4a4a6fbd1d5906f66 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-deactivate, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240507, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:25:42.085 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:42 smithi132 podman[100862]: 2024-05-09 04:25:42.085913285 +0000 UTC m=+0.376795026 container init 33407b9042d9f03e629cc22595faf0071262daa53e4774b4a4a6fbd1d5906f66 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240507, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, maintainer=Guillaume Abrioux ) 2024-05-09T04:25:42.382 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:42 smithi132 podman[100862]: 2024-05-09 04:25:42.08916422 +0000 UTC m=+0.380045974 container start 33407b9042d9f03e629cc22595faf0071262daa53e4774b4a4a6fbd1d5906f66 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-09T04:25:42.382 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:42 smithi132 podman[100862]: 2024-05-09 04:25:42.152371473 +0000 UTC m=+0.443253233 container attach 33407b9042d9f03e629cc22595faf0071262daa53e4774b4a4a6fbd1d5906f66 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, ceph=True) 2024-05-09T04:25:42.752 INFO:journalctl@ceph.osd.7.smithi132.stdout:May 09 04:25:42 smithi132 podman[100862]: 2024-05-09 04:25:42.383662407 +0000 UTC m=+0.674544151 container died 33407b9042d9f03e629cc22595faf0071262daa53e4774b4a4a6fbd1d5906f66 (image=quay.ceph.io/ceph-ci/ceph@sha256:05de6593b88417234cfc69df4d6993be606178f15beb2e23381c5fdef15b9082, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-osd-7-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-05-08-1927-c66c553, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=627304247bdbcb1f80af6cad8aac9a1ee2979504, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240507, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-05-09T04:25:42.907 DEBUG:teuthology.orchestra.run.smithi132:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@osd.7.service' 2024-05-09T04:25:42.945 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:25:42.945 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-05-09T04:25:42.945 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-05-09T04:25:42.945 DEBUG:teuthology.orchestra.run.smithi132:> sudo systemctl stop ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a 2024-05-09T04:25:43.253 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 systemd[1]: Stopping Ceph prometheus.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:25:43.558 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:43.282Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-09T04:25:43.558 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:43.282Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-05-09T04:25:43.559 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:43.282Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-05-09T04:25:43.559 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:43.282Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-09T04:25:43.559 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:43.282Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-05-09T04:25:43.559 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:43.282Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-05-09T04:25:43.559 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:43.282Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-05-09T04:25:43.559 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:43.282Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-05-09T04:25:43.559 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:43.282Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-05-09T04:25:43.559 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:43.283Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-05-09T04:25:43.559 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:43.283Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-05-09T04:25:43.559 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a[62510]: ts=2024-05-09T04:25:43.283Z caller=main.go:1147 level=info msg="See you next time!" 2024-05-09T04:25:43.559 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 podman[100966]: 2024-05-09 04:25:43.292285104 +0000 UTC m=+0.126481497 container died c02ff059e70485a9cb9db9cd016302432392369689621c6e354a4b1d9917d692 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:25:43.942 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 podman[100966]: 2024-05-09 04:25:43.667094641 +0000 UTC m=+0.501291032 container cleanup c02ff059e70485a9cb9db9cd016302432392369689621c6e354a4b1d9917d692 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-09T04:25:43.942 INFO:journalctl@ceph.prometheus.a.smithi132.stdout:May 09 04:25:43 smithi132 bash[100966]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-prometheus-a 2024-05-09T04:25:44.011 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:43.558Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=9 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi078.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.78:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi132.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.132:8443: connect: connection refused" 2024-05-09T04:25:44.011 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:43.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi078.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.78:8443: connect: connection refused" 2024-05-09T04:25:44.011 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:43 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:43.559Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi132.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.132:8443: connect: connection refused" 2024-05-09T04:25:44.110 DEBUG:teuthology.orchestra.run.smithi132:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2777c79e-0db6-11ef-bc97-c7b262605968@prometheus.a.service' 2024-05-09T04:25:44.144 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-09T04:25:44.144 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-05-09T04:25:44.144 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 --force --keep-logs 2024-05-09T04:25:46.011 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:25:45 smithi078 systemd[1]: Stopping Ceph node-exporter.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:25:46.011 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:25:45 smithi078 podman[132094]: 2024-05-09 04:25:45.865747998 +0000 UTC m=+0.104973097 container died c0004282e7ce84fe0f2052628381fcac473b42c0f4d37c197764d40ddc2089ec (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T04:25:46.396 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:25:46 smithi078 podman[132094]: 2024-05-09 04:25:46.204938073 +0000 UTC m=+0.444163161 container cleanup c0004282e7ce84fe0f2052628381fcac473b42c0f4d37c197764d40ddc2089ec (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T04:25:46.396 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:25:46 smithi078 bash[132094]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a 2024-05-09T04:25:46.649 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:25:46 smithi078 podman[132108]: 2024-05-09 04:25:46.397792529 +0000 UTC m=+0.531387990 container remove c0004282e7ce84fe0f2052628381fcac473b42c0f4d37c197764d40ddc2089ec (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-09T04:25:46.649 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:25:46 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-05-09T04:25:46.649 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:25:46 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-05-09T04:25:46.649 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:25:46 smithi078 systemd[1]: Stopped Ceph node-exporter.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:25:46.649 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:May 09 04:25:46 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.a.service: Consumed 3.126s CPU time. 2024-05-09T04:25:47.010 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:46 smithi078 systemd[1]: Stopping Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:25:47.401 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:47 smithi078 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a[78333]: ts=2024-05-09T04:25:47.050Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-05-09T04:25:47.401 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:47 smithi078 podman[132215]: 2024-05-09 04:25:47.091821701 +0000 UTC m=+0.135692623 container died 3cd8b265bc1ddaf746671324335ed79ceb5f7ade0f2d3f772d90d79582f3bcb8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T04:25:47.718 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:47 smithi078 podman[132215]: 2024-05-09 04:25:47.493324392 +0000 UTC m=+0.537195376 container cleanup 3cd8b265bc1ddaf746671324335ed79ceb5f7ade0f2d3f772d90d79582f3bcb8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T04:25:47.718 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:47 smithi078 bash[132215]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a 2024-05-09T04:25:47.968 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:47 smithi078 podman[132229]: 2024-05-09 04:25:47.719406323 +0000 UTC m=+0.622357364 container remove 3cd8b265bc1ddaf746671324335ed79ceb5f7ade0f2d3f772d90d79582f3bcb8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-09T04:25:47.969 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:47 smithi078 podman[132229]: 2024-05-09 04:25:47.803222417 +0000 UTC m=+0.706173456 volume remove 2ac6bf7960f2b23a9d7e4e49a421a6b60f6fd4ddce90a1073e47070c29cd6f2c 2024-05-09T04:25:47.969 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:47 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-09T04:25:47.969 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:47 smithi078 systemd[1]: Stopped Ceph alertmanager.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:25:47.969 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:May 09 04:25:47 smithi078 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@alertmanager.a.service: Consumed 1.602s CPU time. 2024-05-09T04:26:10.989 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 --force --keep-logs 2024-05-09T04:26:12.667 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:26:12 smithi132 systemd[1]: Stopping Ceph node-exporter.b for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:26:13.002 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:26:12 smithi132 podman[101364]: 2024-05-09 04:26:12.662938021 +0000 UTC m=+0.124089381 container died 71aa9559f36fb97f3325669dd846883aefe83d02d775bf83100b09d0db5fddb9 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T04:26:13.368 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:26:13 smithi132 podman[101364]: 2024-05-09 04:26:13.168594238 +0000 UTC m=+0.629745592 container cleanup 71aa9559f36fb97f3325669dd846883aefe83d02d775bf83100b09d0db5fddb9 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T04:26:13.368 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:26:13 smithi132 bash[101364]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b 2024-05-09T04:26:13.648 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:26:13 smithi132 podman[101378]: 2024-05-09 04:26:13.369753949 +0000 UTC m=+0.707285571 container remove 71aa9559f36fb97f3325669dd846883aefe83d02d775bf83100b09d0db5fddb9 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-09T04:26:13.648 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:26:13 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-05-09T04:26:13.648 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:26:13 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-05-09T04:26:13.648 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:26:13 smithi132 systemd[1]: Stopped Ceph node-exporter.b for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:26:13.648 INFO:journalctl@ceph.node-exporter.b.smithi132.stdout:May 09 04:26:13 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@node-exporter.b.service: Consumed 3.215s CPU time. 2024-05-09T04:26:13.907 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:26:13 smithi132 systemd[1]: Stopping Ceph grafana.a for 2777c79e-0db6-11ef-bc97-c7b262605968... 2024-05-09T04:26:13.907 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:26:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=server t=2024-05-09T04:26:13.898091139Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-05-09T04:26:13.907 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:26:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=tracing t=2024-05-09T04:26:13.898486461Z level=info msg="Closing tracing" 2024-05-09T04:26:13.908 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:26:13 smithi132 ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a[64722]: logger=ticker t=2024-05-09T04:26:13.898542574Z level=info msg=stopped last_tick=2024-05-09T04:26:10Z 2024-05-09T04:26:14.172 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:26:13 smithi132 podman[101489]: 2024-05-09 04:26:13.908506682 +0000 UTC m=+0.109554111 container died 663d68ed82f1d37c3e0922fe9df5a0554d9fbac8d0cc0dae68686069b28e7f25 (image=quay.io/ceph/grafana:9.4.12, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-09T04:26:14.503 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:26:14 smithi132 podman[101489]: 2024-05-09 04:26:14.239621795 +0000 UTC m=+0.440669236 container cleanup 663d68ed82f1d37c3e0922fe9df5a0554d9fbac8d0cc0dae68686069b28e7f25 (image=quay.io/ceph/grafana:9.4.12, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-09T04:26:14.503 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:26:14 smithi132 bash[101489]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a 2024-05-09T04:26:14.503 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:26:14 smithi132 podman[101509]: 2024-05-09 04:26:14.415807986 +0000 UTC m=+0.508722063 container remove 663d68ed82f1d37c3e0922fe9df5a0554d9fbac8d0cc0dae68686069b28e7f25 (image=quay.io/ceph/grafana:9.4.12, name=ceph-2777c79e-0db6-11ef-bc97-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-09T04:26:15.002 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:26:14 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@grafana.a.service: Deactivated successfully. 2024-05-09T04:26:15.003 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:26:14 smithi132 systemd[1]: Stopped Ceph grafana.a for 2777c79e-0db6-11ef-bc97-c7b262605968. 2024-05-09T04:26:15.003 INFO:journalctl@ceph.grafana.a.smithi132.stdout:May 09 04:26:14 smithi132 systemd[1]: ceph-2777c79e-0db6-11ef-bc97-c7b262605968@grafana.a.service: Consumed 12.107s CPU time. 2024-05-09T04:26:26.407 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-09T04:26:26.441 DEBUG:teuthology.orchestra.run.smithi132:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-09T04:26:26.468 INFO:tasks.cephadm:Archiving crash dumps... 2024-05-09T04:26:26.470 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/crash to /home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492/remote/smithi078/crash 2024-05-09T04:26:26.471 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/crash -- . 2024-05-09T04:26:26.526 INFO:teuthology.orchestra.run.smithi078.stderr:tar: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/crash: Cannot open: No such file or directory 2024-05-09T04:26:26.527 INFO:teuthology.orchestra.run.smithi078.stderr:tar: Error is not recoverable: exiting now 2024-05-09T04:26:26.529 DEBUG:teuthology.misc:Transferring archived files from smithi132:/var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/crash to /home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492/remote/smithi132/crash 2024-05-09T04:26:26.529 DEBUG:teuthology.orchestra.run.smithi132:> sudo tar c -f - -C /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/crash -- . 2024-05-09T04:26:26.564 INFO:teuthology.orchestra.run.smithi132.stderr:tar: /var/lib/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/crash: Cannot open: No such file or directory 2024-05-09T04:26:26.564 INFO:teuthology.orchestra.run.smithi132.stderr:tar: Error is not recoverable: exiting now 2024-05-09T04:26:26.565 INFO:tasks.cephadm:Checking cluster log for badness... 2024-05-09T04:26:26.565 DEBUG:teuthology.orchestra.run.smithi078:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_STRAY_DAEMON | head -n 1 2024-05-09T04:26:26.594 INFO:teuthology.orchestra.run.smithi078.stdout:2024-05-09T04:00:19.068007+0000 mon.a (mon.0) 928 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi078.vnplzc on smithi078 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-05-09T04:26:26.595 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-05-09T04:26:26.595 DEBUG:teuthology.orchestra.run.smithi078:> sudo egrep '\[SEC\]' /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_STRAY_DAEMON | head -n 1 2024-05-09T04:26:26.663 DEBUG:teuthology.orchestra.run.smithi078:> sudo egrep '\[ERR\]' /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_STRAY_DAEMON | head -n 1 2024-05-09T04:26:26.731 DEBUG:teuthology.orchestra.run.smithi078:> sudo egrep '\[WRN\]' /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_STRAY_DAEMON | head -n 1 2024-05-09T04:26:26.797 INFO:teuthology.orchestra.run.smithi078.stdout:2024-05-09T04:00:19.068007+0000 mon.a (mon.0) 928 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi078.vnplzc on smithi078 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-05-09T04:26:26.798 INFO:tasks.cephadm:Compressing logs... 2024-05-09T04:26:26.798 DEBUG:teuthology.orchestra.run.smithi078:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-09T04:26:26.840 DEBUG:teuthology.orchestra.run.smithi132:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-09T04:26:26.862 INFO:teuthology.orchestra.run.smithi078.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-09T04:26:26.862 INFO:teuthology.orchestra.run.smithi078.stderr:: No such file or directory 2024-05-09T04:26:26.862 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.log 2024-05-09T04:26:26.864 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5/var/log/ceph/cephadm.log: --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.2.log 2024-05-09T04:26:26.865 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.audit.log 2024-05-09T04:26:26.865 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.log: /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.3.log 2024-05-09T04:26:26.865 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.1.log 2024-05-09T04:26:26.865 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.audit.log: /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mgr.y.log 2024-05-09T04:26:26.865 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.0.log 2024-05-09T04:26:26.865 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mon.c.log 2024-05-09T04:26:26.866 INFO:teuthology.orchestra.run.smithi132.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-09T04:26:26.866 INFO:teuthology.orchestra.run.smithi132.stderr:: No such file or directory 2024-05-09T04:26:26.866 INFO:teuthology.orchestra.run.smithi132.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.log 2024-05-09T04:26:26.866 INFO:teuthology.orchestra.run.smithi132.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.audit.log 2024-05-09T04:26:26.866 INFO:teuthology.orchestra.run.smithi132.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.7.log 2024-05-09T04:26:26.868 INFO:teuthology.orchestra.run.smithi132.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.5.log 2024-05-09T04:26:26.868 INFO:teuthology.orchestra.run.smithi132.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.4.log 2024-05-09T04:26:26.868 INFO:teuthology.orchestra.run.smithi132.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.audit.log: /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.7.log: /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mgr.x.log 2024-05-09T04:26:26.869 INFO:teuthology.orchestra.run.smithi132.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-client.rgw.foo.smithi132.pcutwm.log 2024-05-09T04:26:26.869 INFO:teuthology.orchestra.run.smithi132.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.6.log 2024-05-09T04:26:26.869 INFO:teuthology.orchestra.run.smithi132.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-client.rgw.foo.smithi132.pcutwm.log: 75.5% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-client.rgw.foo.smithi132.pcutwm.log.gz 2024-05-09T04:26:26.869 INFO:teuthology.orchestra.run.smithi132.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.cephadm.log 2024-05-09T04:26:26.870 INFO:teuthology.orchestra.run.smithi132.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.6.log: 89.1% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.log.gz 2024-05-09T04:26:26.870 INFO:teuthology.orchestra.run.smithi132.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-volume.log 2024-05-09T04:26:26.871 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.0.log: 94.0% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.log.gz 2024-05-09T04:26:26.871 INFO:teuthology.orchestra.run.smithi132.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.cephadm.log: 90.3% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.cephadm.log.gz 2024-05-09T04:26:26.873 INFO:teuthology.orchestra.run.smithi132.stderr: 91.1% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.audit.log.gz 2024-05-09T04:26:26.873 INFO:teuthology.orchestra.run.smithi078.stderr: 94.4% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.audit.log.gz 2024-05-09T04:26:26.874 INFO:teuthology.orchestra.run.smithi132.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mon.b.log 2024-05-09T04:26:26.878 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mon.a.log 2024-05-09T04:26:26.878 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.cephadm.log 2024-05-09T04:26:26.879 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mon.a.log: 92.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-09T04:26:26.889 INFO:teuthology.orchestra.run.smithi132.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-volume.log: /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mon.b.log: 91.0% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-volume.log.gz 2024-05-09T04:26:26.892 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/tcmu-runner.log 2024-05-09T04:26:26.894 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.cephadm.log: 94.2% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph.cephadm.log.gz 2024-05-09T04:26:26.900 INFO:teuthology.orchestra.run.smithi132.stderr: 90.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-09T04:26:26.905 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-volume.log 2024-05-09T04:26:26.905 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/tcmu-runner.log: 89.5% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/tcmu-runner.log.gz 2024-05-09T04:26:26.918 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-client.rgw.foo.smithi078.bfidtv.log 2024-05-09T04:26:26.931 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-volume.log: /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-client.rgw.foo.smithi078.bfidtv.log: 75.9% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-client.rgw.foo.smithi078.bfidtv.log.gz 2024-05-09T04:26:26.933 INFO:teuthology.orchestra.run.smithi078.stderr: 91.0% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-volume.log.gz 2024-05-09T04:26:27.025 INFO:teuthology.orchestra.run.smithi132.stderr: 89.7% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mgr.x.log.gz 2024-05-09T04:26:27.315 INFO:teuthology.orchestra.run.smithi078.stderr: 89.1% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mgr.y.log.gz 2024-05-09T04:26:27.649 INFO:teuthology.orchestra.run.smithi078.stderr: 92.4% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mon.c.log.gz 2024-05-09T04:26:27.673 INFO:teuthology.orchestra.run.smithi132.stderr: 92.5% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mon.b.log.gz 2024-05-09T04:26:28.253 INFO:teuthology.orchestra.run.smithi078.stderr: 93.3% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.2.log.gz 2024-05-09T04:26:28.356 INFO:teuthology.orchestra.run.smithi132.stderr: 93.2% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.6.log.gz 2024-05-09T04:26:28.547 INFO:teuthology.orchestra.run.smithi078.stderr: 91.0% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-mon.a.log.gz 2024-05-09T04:26:28.772 INFO:teuthology.orchestra.run.smithi132.stderr: 93.3% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.5.log.gz 2024-05-09T04:26:28.774 INFO:teuthology.orchestra.run.smithi078.stderr: 93.4% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.1.log.gz 2024-05-09T04:26:29.011 INFO:teuthology.orchestra.run.smithi132.stderr: 93.8% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.7.log.gz 2024-05-09T04:26:29.088 INFO:teuthology.orchestra.run.smithi132.stderr: 93.5% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.4.log.gz 2024-05-09T04:26:29.091 INFO:teuthology.orchestra.run.smithi132.stderr: 2024-05-09T04:26:29.091 INFO:teuthology.orchestra.run.smithi132.stderr:real 0m2.236s 2024-05-09T04:26:29.091 INFO:teuthology.orchestra.run.smithi132.stderr:user 0m8.461s 2024-05-09T04:26:29.091 INFO:teuthology.orchestra.run.smithi132.stderr:sys 0m0.331s 2024-05-09T04:26:29.099 INFO:teuthology.orchestra.run.smithi078.stderr: 93.4% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.0.log.gz 2024-05-09T04:26:29.637 INFO:teuthology.orchestra.run.smithi078.stderr: 93.4% -- replaced with /var/log/ceph/2777c79e-0db6-11ef-bc97-c7b262605968/ceph-osd.3.log.gz 2024-05-09T04:26:29.639 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-05-09T04:26:29.639 INFO:teuthology.orchestra.run.smithi078.stderr:real 0m2.786s 2024-05-09T04:26:29.639 INFO:teuthology.orchestra.run.smithi078.stderr:user 0m10.848s 2024-05-09T04:26:29.639 INFO:teuthology.orchestra.run.smithi078.stderr:sys 0m0.408s 2024-05-09T04:26:29.639 INFO:tasks.cephadm:Archiving logs... 2024-05-09T04:26:29.640 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/log/ceph to /home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492/remote/smithi078/log 2024-05-09T04:26:29.641 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-09T04:26:31.225 DEBUG:teuthology.misc:Transferring archived files from smithi132:/var/log/ceph to /home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492/remote/smithi132/log 2024-05-09T04:26:31.226 DEBUG:teuthology.orchestra.run.smithi132:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-09T04:26:32.438 INFO:tasks.cephadm:Removing cluster... 2024-05-09T04:26:32.438 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 --force 2024-05-09T04:26:32.731 DEBUG:teuthology.orchestra.run.smithi132:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2777c79e-0db6-11ef-bc97-c7b262605968 --force 2024-05-09T04:26:33.025 INFO:tasks.cephadm:Removing cephadm ... 2024-05-09T04:26:33.025 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-09T04:26:33.041 DEBUG:teuthology.orchestra.run.smithi132:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-09T04:26:33.056 INFO:tasks.cephadm:Teardown complete 2024-05-09T04:26:33.056 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-05-09T04:26:33.066 INFO:teuthology.task.clock:Checking final clock skew... 2024-05-09T04:26:33.067 DEBUG:teuthology.orchestra.run.smithi078:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-09T04:26:33.084 DEBUG:teuthology.orchestra.run.smithi132:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-09T04:26:33.096 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-05-09T04:26:33.100 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-09T04:26:33.100 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-05-09T04:26:33.100 INFO:teuthology.orchestra.run.smithi078.stdout:^* hv01.front.sepia.ceph.com 3 6 377 53 +3034us[+3047us] +/- 55ms 2024-05-09T04:26:33.100 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 55 -1504us[-1491us] +/- 66ms 2024-05-09T04:26:33.100 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv03.front.sepia.ceph.com 3 8 377 185 -1992us[-1735us] +/- 72ms 2024-05-09T04:26:33.100 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-09T04:26:33.110 INFO:teuthology.orchestra.run.smithi132.stderr:bash: line 1: ntpq: command not found 2024-05-09T04:26:33.115 INFO:teuthology.orchestra.run.smithi132.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-09T04:26:33.115 INFO:teuthology.orchestra.run.smithi132.stdout:=============================================================================== 2024-05-09T04:26:33.115 INFO:teuthology.orchestra.run.smithi132.stdout:^+ hv01.front.sepia.ceph.com 3 6 377 50 +3091us[+3091us] +/- 55ms 2024-05-09T04:26:33.115 INFO:teuthology.orchestra.run.smithi132.stdout:^* hv02.front.sepia.ceph.com 3 6 377 51 -1452us[-1229us] +/- 66ms 2024-05-09T04:26:33.115 INFO:teuthology.orchestra.run.smithi132.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 51 -1971us[-1748us] +/- 74ms 2024-05-09T04:26:33.115 INFO:teuthology.orchestra.run.smithi132.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-09T04:26:33.116 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-05-09T04:26:33.125 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-05-09T04:26:33.125 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-05-09T04:26:33.134 DEBUG:teuthology.orchestra.run.smithi078:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-09T04:26:33.228 DEBUG:teuthology.orchestra.run.smithi132:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-09T04:26:33.303 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-09T04:26:33.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-09T04:26:33.332 DEBUG:teuthology.orchestra.run.smithi132:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-09T04:26:33.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-09T04:26:33.360 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-05-09T04:26:33.370 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-05-09T04:26:33.378 INFO:teuthology.task.internal:Duration was 2883.828238 seconds 2024-05-09T04:26:33.378 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-05-09T04:26:33.387 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-05-09T04:26:33.387 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-09T04:26:33.389 DEBUG:teuthology.orchestra.run.smithi132:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-09T04:26:33.425 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-09T04:26:33.439 INFO:teuthology.orchestra.run.smithi132.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-09T04:26:33.799 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-05-09T04:26:33.799 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi078.front.sepia.ceph.com 2024-05-09T04:26:33.800 DEBUG:teuthology.orchestra.run.smithi078:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-09T04:26:33.860 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi132.front.sepia.ceph.com 2024-05-09T04:26:33.860 DEBUG:teuthology.orchestra.run.smithi132:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-09T04:26:33.893 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-05-09T04:26:33.893 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-09T04:26:33.903 DEBUG:teuthology.orchestra.run.smithi132:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-09T04:26:33.959 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-05-09T04:26:33.960 DEBUG:teuthology.orchestra.run.smithi078:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-09T04:26:33.970 DEBUG:teuthology.orchestra.run.smithi132:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-09T04:26:34.585 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-05-09T04:26:34.595 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-05-09T04:26:34.595 DEBUG:teuthology.orchestra.run.smithi078:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-09T04:26:34.620 DEBUG:teuthology.orchestra.run.smithi132:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-09T04:26:34.645 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-05-09T04:26:34.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-09T04:26:34.662 DEBUG:teuthology.orchestra.run.smithi132:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-09T04:26:34.682 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = core 2024-05-09T04:26:34.709 INFO:teuthology.orchestra.run.smithi132.stdout:kernel.core_pattern = core 2024-05-09T04:26:34.746 DEBUG:teuthology.orchestra.run.smithi078:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-09T04:26:34.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-09T04:26:34.767 DEBUG:teuthology.orchestra.run.smithi132:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-09T04:26:34.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-09T04:26:34.801 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-05-09T04:26:34.811 INFO:teuthology.task.internal:Transferring archived files... 2024-05-09T04:26:34.812 DEBUG:teuthology.misc:Transferring archived files from smithi078:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492/remote/smithi078 2024-05-09T04:26:34.812 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-09T04:26:34.861 DEBUG:teuthology.misc:Transferring archived files from smithi132:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492/remote/smithi132 2024-05-09T04:26:34.861 DEBUG:teuthology.orchestra.run.smithi132:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-09T04:26:34.900 INFO:teuthology.task.internal:Removing archive directory... 2024-05-09T04:26:34.901 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-09T04:26:34.903 DEBUG:teuthology.orchestra.run.smithi132:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-09T04:26:34.949 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-05-09T04:26:34.961 INFO:teuthology.task.internal:Not uploading archives. 2024-05-09T04:26:34.961 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-05-09T04:26:34.971 INFO:teuthology.task.internal:Tidying up after the test... 2024-05-09T04:26:34.971 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-09T04:26:34.973 DEBUG:teuthology.orchestra.run.smithi132:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-09T04:26:34.986 INFO:teuthology.orchestra.run.smithi078.stdout: 265615 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 9 04:26 /home/ubuntu/cephtest 2024-05-09T04:26:35.004 INFO:teuthology.orchestra.run.smithi132.stdout: 262165 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 9 04:26 /home/ubuntu/cephtest 2024-05-09T04:26:35.005 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-05-09T04:26:35.016 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-05-09T04:26:35.036 INFO:teuthology.nuke:Checking targets against current locks 2024-05-09T04:26:35.065 DEBUG:teuthology.nuke:shortname: smithi078 2024-05-09T04:26:35.065 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-09T04:26:35.089 DEBUG:teuthology.nuke:shortname: smithi132 2024-05-09T04:26:35.089 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-09T04:26:35.093 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi078.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-09 03:27:28.702707', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHcBM+SsFuPZS1hO0Jlswm+6Gw/1lbNPUcdD03IbpUhCcYq+Vo0k7qQ1qrFnJdZGFfYSEMkrONbPu9h78DjztIw='} 2024-05-09T04:26:35.116 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi132.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-05-09_03:09:40-orch:cephadm-wip-adk-testing-2024-05-08-1927-distro-default-smithi/7699492', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-09 03:27:28.701300', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKootgy+oHLu3wh4K4h2ucOE2hhkSS/TIF9xEI1QeTZFzii8lw02Hf0mTkQT/JEobk6WkJWeLNWWEdVIzburuRc='} 2024-05-09T04:26:35.138 INFO:teuthology.orchestra.console.smithi078:Power off 2024-05-09T04:26:35.138 DEBUG:teuthology.orchestra.console.smithi078:pexpect command: ipmitool -H smithi078.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-05-09T04:26:35.162 DEBUG:teuthology.orchestra.console.smithi078:power off output: Chassis Power Control: Down/Off 2024-05-09T04:26:35.162 DEBUG:teuthology.orchestra.console.smithi078:pexpect command: ipmitool -H smithi078.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-09T04:26:35.177 INFO:teuthology.orchestra.console.smithi132:Power off 2024-05-09T04:26:35.177 DEBUG:teuthology.orchestra.console.smithi132:pexpect command: ipmitool -H smithi132.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-05-09T04:26:35.186 DEBUG:teuthology.orchestra.console.smithi078:check power output: Chassis Power is on 2024-05-09T04:26:35.201 DEBUG:teuthology.orchestra.console.smithi132:power off output: Chassis Power Control: Down/Off 2024-05-09T04:26:35.201 DEBUG:teuthology.orchestra.console.smithi132:pexpect command: ipmitool -H smithi132.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-09T04:26:35.224 DEBUG:teuthology.orchestra.console.smithi132:check power output: Chassis Power is on 2024-05-09T04:26:39.188 DEBUG:teuthology.orchestra.console.smithi078:pexpect command: ipmitool -H smithi078.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-09T04:26:39.225 DEBUG:teuthology.orchestra.console.smithi132:pexpect command: ipmitool -H smithi132.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-09T04:26:39.301 DEBUG:teuthology.orchestra.console.smithi078:check power output: Chassis Power is on 2024-05-09T04:26:39.337 DEBUG:teuthology.orchestra.console.smithi132:check power output: Chassis Power is on 2024-05-09T04:26:43.305 DEBUG:teuthology.orchestra.console.smithi078:pexpect command: ipmitool -H smithi078.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-09T04:26:43.338 DEBUG:teuthology.orchestra.console.smithi132:pexpect command: ipmitool -H smithi132.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-09T04:26:43.417 DEBUG:teuthology.orchestra.console.smithi078:check power output: Chassis Power is off 2024-05-09T04:26:43.448 DEBUG:teuthology.orchestra.console.smithi132:check power output: Chassis Power is off 2024-05-09T04:26:43.518 INFO:teuthology.orchestra.console.smithi078:Power off completed 2024-05-09T04:26:43.550 INFO:teuthology.orchestra.console.smithi132:Power off completed 2024-05-09T04:26:43.654 INFO:teuthology.run:Summary data: description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 2883.8282384872437 failure_reason: '"2024-05-09T04:00:19.068007+0000 mon.a (mon.0) 928 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) [''daemon iscsi.foo.smithi078.vnplzc on smithi078 is in unknown state''] (CEPHADM_FAILED_DAEMON)" in cluster log' owner: scheduled_adking@teuthology success: false 2024-05-09T04:26:43.654 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-09T04:26:43.727 INFO:teuthology.run:FAIL